From f37493dbd76dfd68c9ec985d0a364f16b0e385c8 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 23 Sep 2026 20:35:18 -0400 Subject: [PATCH 01/18] add planet mapmaker database --- mapcat/database/__init__.py | 6 ++ mapcat/database/planet_map.py | 176 +++++++++++++++++++++++++++++++ mapcat/database/planet_todfit.py | 120 +++++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 mapcat/database/planet_map.py create mode 100644 mapcat/database/planet_todfit.py diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index 6839745..94a6b16 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -11,6 +11,8 @@ from .sky_coverage import SkyCoverageTable from .time_domain_processing import TimeDomainProcessingTable from .tod import TODDepthOneTable +from .planet_map import PlanetMapTable +from .planet_todfit import PlanetTodFitTable __all__ = [ "AtomicMapCoaddTable", @@ -22,6 +24,8 @@ "SkyCoverageTable", "TODDepthOneTable", "TimeDomainProcessingTable", + "PlanetMapTable", + "PlanetTodFitTable", ] ALL_TABLES = [ @@ -34,4 +38,6 @@ TODDepthOneTable, PipelineInformationTable, SkyCoverageTable, + PlanetMapTable, + PlanetTodFitTable, ] diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py new file mode 100644 index 0000000..e45cde3 --- /dev/null +++ b/mapcat/database/planet_map.py @@ -0,0 +1,176 @@ +""" +Table for planet maps. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any +import json + +from astropy.time import Time +from astropydantic import AstroPydanticTime +from sqlmodel import Field, Relationship, SQLModel +from sqlalchemy import Column, JSON + +class PlanetMap(SQLModel): + obs_id: str + telescope: str + freq_channel: str + wafer: str + ctime: AstroPydanticTime + dtime: datetime + source: str + + hit_path: str | None + map_path: str | None + weight_path: str | None + weighted_map_path: str | None + + azimuth: float | None + elevation: float | None + duration: float| None + pwv: float | None + pwv_std: float | None + pwv_p2p: float | None + pwv_apex: float | None + pwv_apex_std: float | None + pwv_apex_p2p: float | None + + f_hwp: float | None + roll_angle: float | None + scan_speed: float | None + scan_acc: float | None + sun_distance: float | None + moon_distance: float | None + wind_speed: float | None + wind_direction: float | None + ambient_temperature: float | None + uv: float | None + + detnum_before_fitselection: int | None + total_detnum: int | None + recenter: bool | None + yc: float | None + xc: float | None + + Tmap_variance: float | None + Qmap_variance: float | None + Umap_variance: float | None + + proc: dict[str, Any] | list[Any] | None = None + detnum: list[int] | None = None + detid: list[str] | None = None + + +class PlanetMapTable(SQLModel, table=True): + __tablename__ = "planet_map" + + obs_id: str = Field(primary_key=True) + telescope: str = Field(primary_key=True) + freq_channel: str = Field(primary_key=True) + wafer: str = Field(primary_key=True) + ctime: float = Field(nullable=False, primary_key=True) + source: str = Field(primary_key=True) + dtime: datetime = Field() + + hit_path: str | None = Field() + map_path: str | None = Field() + weight_path: str | None = Field() + weighted_map_path: str | None = Field() + + elevation: float | None = Field() + duration: float | None = Field() + azimuth: float | None = Field() + pwv: float | None = Field() + pwv_std: float | None = Field() + pwv_p2p: float | None = Field() + pwv_apex: float | None = Field() + pwv_apex_std: float | None = Field() + pwv_apex_p2p: float | None = Field() + f_hwp: float | None = Field() + roll_angle: float | None = Field() + scan_speed: float | None = Field() + scan_acc: float | None = Field() + sun_distance: float | None = Field() + moon_distance: float | None = Field() + wind_speed: float | None = Field() + wind_direction: float | None = Field() + ambient_temperature: float | None = Field() + uv: float | None = Field() + + detnum_before_fitselection: int | None = Field() + total_detnum: int | None = Field() + recenter: bool | None = Field() + yc: float | None = Field() + xc: float | None = Field() + + Tmap_variance: float | None = Field() + Qmap_variance: float | None = Field() + Umap_variance: float | None = Field() + + proc: dict[str, Any] | list[Any] | None = Field( + default=None, + sa_column=Column(JSON, nullable=True), + ) + detnum: list[int] | None = Field( + default=None, + sa_column=Column(JSON, nullable=True), + ) + detid: list[str] | None = Field( + default=None, + sa_column=Column(JSON, nullable=True), + ) + + + + def to_model(self) -> PlanetMap: + """ + Return an PlanetMap model from this table entry. + + Returns + ------- + PlanetMap : PlanetMap + The PlanetMap model corresponding to this table entry. + """ + return PlanetMap( + obs_id=self.obs_id, + telescope=self.telescope, + freq_channel=self.freq_channel, + wafer=self.wafer, + ctime=Time(self.ctime, format="unix", scale="utc"), + dtime=self.dtime, + source=self.source, + hit_path=self.hit_path, + map_path=self.map_path, + weight_path=self.weight_path, + weighted_map_path=self.weight_map_path, + azimuth=self.azimuth, + elevation=self.elevation, + duration=self.duration, + pwv=self.pwv, + pwv_std=self.pwv_std, + pwv_p2p=self.pwv_p2p, + pwv_apex=self.pwv_apex, + pwv_apex_std=self.pwv_apex_std, + pwv_apex_p2p=self.pwv_apex_p2p, + f_hwp=self.f_hwp, + roll_angle=self.roll_angle, + scan_speed=self.scan_speed, + scan_acc=self.scan_acc, + sun_distance=self.sun_distance, + moon_distance=self.moon_distance, + wind_speed=self.wind_speed, + wind_direction=self.wind_direction, + ambient_temperature=self.ambient_temperature, + uv=self.uv, + detnum_before_fitselection=self.detnum_before_fitselection, + total_detnum=self.total_detnum, + recenter=self.recenter, + yc=self.yc, + xc=self.xc, + Tmap_variance=self.Tmap_variance, + Qmap_variance=self.Qmap_variance, + Umap_variance=self.Umap_variance, + proc=self.proc, + detnum=self.detnum, + detid=self.detid, + ) diff --git a/mapcat/database/planet_todfit.py b/mapcat/database/planet_todfit.py new file mode 100644 index 0000000..8b1ba9a --- /dev/null +++ b/mapcat/database/planet_todfit.py @@ -0,0 +1,120 @@ +""" +Table for planet todfit. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any +import json + +from astropy.time import Time +from astropydantic import AstroPydanticTime +from sqlmodel import Field, Relationship, SQLModel + +class PlanetTodFit(SQLModel): + obs_id: str + telescope: str + freq_channel: str + wafer: str + ctime: AstroPydanticTime + dtime: datetime + source: str + detid: str + + amplitude: float | None + xo: float | None + yo: float | None + sigmax: float | None + sigmay: float | None + theta: float | None + defla: float | None + deflp: float | None + amplitude_err: float | None + xo_err: float | None + yo_err: float | None + sigmax_err: float | None + sigmay_err: float | None + theta_err: float | None + defla_err: float | None + deflp_err: float | None + chisq: float | None + dof: int | None + xi: float | None + eta: float | None + gamma: float | None + + +class PlanetTodFitTable(SQLModel, table=True): + __tablename__ = "planet_todfit" + + obs_id: str = Field(primary_key=True) + telescope: str = Field(primary_key=True) + freq_channel: str = Field(primary_key=True) + wafer: str = Field(primary_key=True) + ctime: float = Field(nullable=False, primary_key=True) + source: str = Field(primary_key=True) + detid: str = Field(primary_key=True) + dtime: datetime = Field() + + amplitude: float | None = Field() + xo: float | None = Field() + yo: float | None = Field() + sigmax: float | None = Field() + sigmay: float | None = Field() + theta: float | None = Field() + defla: float | None = Field() + deflp: float | None = Field() + amplitude_err: float | None = Field() + xo_err: float | None = Field() + yo_err: float | None = Field() + sigmax_err: float | None = Field() + sigmay_err: float | None = Field() + theta_err: float | None = Field() + defla_err: float | None = Field() + deflp_err: float | None = Field() + chisq: float | None = Field() + dof: int | None = Field() + xi: float | None = Field() + eta: float | None = Field() + gamma: float | None = Field() + + + def to_model(self) -> PlanetTodFit: + """ + Return an PlanetTodFit model from this table entry. + + Returns + ------- + PlanetTodFit : PlanetTodFit + The PlanetTodFit model corresponding to this table entry. + """ + return PlanetTodFit( + obs_id=self.obs_id, + telescope=self.telescope, + freq_channel=self.freq_channel, + wafer=self.wafer, + ctime=Time(self.ctime, format="unix", scale="utc"), + dtime=self.dtime, + source=self.source, + detid=self.detid, + amplitude=self.amplitude, + xo=self.xo, + yo=self.yo, + sigmax=self.sigmax, + sigmay=self.sigmay, + theta=self.theta, + defla=self.defla, + deflp=self.deflp, + amplitude_err=self.amplitude_err, + xo_err=self.xo_err, + yo_err=self.yo_err, + sigmax_err=self.sigmax_err, + sigmay_err=self.sigmay_err, + theta_err=self.theta_err, + defla_err=self.defla_err, + deflp_err=self.deflp_err, + chisq=self.chisq, + dof=self.dof, + xi=self.xi, + eta=self.eta, + gamma=self.gamma, + ) From b9eb7b3d2f4489b2e7a29a3bdc5aad778053f3cf Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 23 Sep 2026 21:40:11 -0400 Subject: [PATCH 02/18] add fit result --- mapcat/database/planet_map.py | 62 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index e45cde3..9b19f49 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -59,7 +59,27 @@ class PlanetMap(SQLModel): proc: dict[str, Any] | list[Any] | None = None detnum: list[int] | None = None detid: list[str] | None = None - + # Main beam fit result + amplitude: float | None + peak: float | None + xo: float | None + yo: float | None + sigmax: float | None + sigmay: float | None + theta: float | None + redchit: float | None + # Leakage beam fit result + mq: float | None + d0q: float | None + d1q: float | None + sigmaq: float | None + redchiq: float | None + mu: float | None + d0u: float | None + d1u: float | None + sigmau: float | None + redchiu: float | None + class PlanetMapTable(SQLModel, table=True): __tablename__ = "planet_map" @@ -119,7 +139,27 @@ class PlanetMapTable(SQLModel, table=True): default=None, sa_column=Column(JSON, nullable=True), ) - + # Main beam fit result + peak: float | None = Field() + amplitude: float | None = Field() + xo: float | None = Field() + yo: float | None = Field() + sigmax: float | None = Field() + sigmay: float | None = Field() + theta: float | None = Field() + redchit: float | None = Field() + # Leakage beam fit result + mq: float | None = Field() + d0q: float | None = Field() + d1q: float | None = Field() + sigmaq: float | None = Field() + redchiq: float | None = Field() + mu: float | None = Field() + d0u: float | None = Field() + d1u: float | None = Field() + sigmau: float | None = Field() + redchiu: float | None = Field() + def to_model(self) -> PlanetMap: @@ -173,4 +213,22 @@ def to_model(self) -> PlanetMap: proc=self.proc, detnum=self.detnum, detid=self.detid, + peak=self.peak, + amplitude=self.amplitude, + xo=self.xo, + yo=self.yo, + sigmax=self.sigmax, + sigmay=self.sigmay, + theta=self.theta, + redchit=self.redchit, + mq=self.mq, + d0q=self.d0q, + d1q=self.d1q, + sigmaq=self.sigmaq, + redchiq=self.redchiq, + mu=self.mu, + d0u=self.d0u, + d1u=self.d1u, + sigmau=self.sigmau, + redchiu=self.redchiu, ) From 1e56629867474614dc0c97e3621091b7a7d94584 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 23 Sep 2026 21:45:25 -0400 Subject: [PATCH 03/18] fix a bug --- mapcat/database/planet_map.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index 9b19f49..2445ac6 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -3,12 +3,11 @@ """ from datetime import datetime -from typing import TYPE_CHECKING, Any -import json +from typing import Any from astropy.time import Time from astropydantic import AstroPydanticTime -from sqlmodel import Field, Relationship, SQLModel +from sqlmodel import Field, SQLModel from sqlalchemy import Column, JSON class PlanetMap(SQLModel): From de6612842c6f1e71752b733cb2b627f629833d41 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 23 Sep 2026 21:50:25 -0400 Subject: [PATCH 04/18] fix a bug --- mapcat/database/planet_todfit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mapcat/database/planet_todfit.py b/mapcat/database/planet_todfit.py index 8b1ba9a..ef21055 100644 --- a/mapcat/database/planet_todfit.py +++ b/mapcat/database/planet_todfit.py @@ -3,12 +3,10 @@ """ from datetime import datetime -from typing import TYPE_CHECKING, Any -import json from astropy.time import Time from astropydantic import AstroPydanticTime -from sqlmodel import Field, Relationship, SQLModel +from sqlmodel import Field, SQLModel class PlanetTodFit(SQLModel): obs_id: str From 01676d068e8c045ce73f7179efa4c8226f78affa Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 23 Sep 2026 21:55:36 -0400 Subject: [PATCH 05/18] fix a bug --- mapcat/database/__init__.py | 8 ++++---- mapcat/database/planet_map.py | 3 ++- mapcat/database/planet_todfit.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index 94a6b16..d0cd7a3 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -7,12 +7,12 @@ from .depth_one_coadd import DepthOneCoaddTable from .depth_one_map import DepthOneMapTable from .pipeline_information import PipelineInformationTable +from .planet_map import PlanetMapTable +from .planet_todfit import PlanetTodFitTable from .pointing_residual import PointingResidualTable from .sky_coverage import SkyCoverageTable from .time_domain_processing import TimeDomainProcessingTable from .tod import TODDepthOneTable -from .planet_map import PlanetMapTable -from .planet_todfit import PlanetTodFitTable __all__ = [ "AtomicMapCoaddTable", @@ -20,12 +20,12 @@ "DepthOneCoaddTable", "DepthOneMapTable", "PipelineInformationTable", + "PlanetMapTable", + "PlanetTodFitTable", "PointingResidualTable", "SkyCoverageTable", "TODDepthOneTable", "TimeDomainProcessingTable", - "PlanetMapTable", - "PlanetTodFitTable", ] ALL_TABLES = [ diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index 2445ac6..ae933f2 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -7,8 +7,9 @@ from astropy.time import Time from astropydantic import AstroPydanticTime +from sqlalchemy import JSON, Column from sqlmodel import Field, SQLModel -from sqlalchemy import Column, JSON + class PlanetMap(SQLModel): obs_id: str diff --git a/mapcat/database/planet_todfit.py b/mapcat/database/planet_todfit.py index ef21055..4e9a26b 100644 --- a/mapcat/database/planet_todfit.py +++ b/mapcat/database/planet_todfit.py @@ -8,6 +8,7 @@ from astropydantic import AstroPydanticTime from sqlmodel import Field, SQLModel + class PlanetTodFit(SQLModel): obs_id: str telescope: str From f1b22e09eb6ca2bd9ed90eb42b49b31b2a0f17c0 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Mon, 28 Sep 2026 21:16:57 -0400 Subject: [PATCH 06/18] update database class --- mapcat/database/__init__.py | 4 +- mapcat/database/planet_map.py | 231 ++++++++++++++++++++++++------- mapcat/database/planet_todfit.py | 5 +- 3 files changed, 188 insertions(+), 52 deletions(-) diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index d0cd7a3..a09e5c4 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -7,7 +7,7 @@ from .depth_one_coadd import DepthOneCoaddTable from .depth_one_map import DepthOneMapTable from .pipeline_information import PipelineInformationTable -from .planet_map import PlanetMapTable +from .planet_map import PlanetMapTable, PlanetMapFitTable from .planet_todfit import PlanetTodFitTable from .pointing_residual import PointingResidualTable from .sky_coverage import SkyCoverageTable @@ -21,6 +21,7 @@ "DepthOneMapTable", "PipelineInformationTable", "PlanetMapTable", + "PlanetMapFitTable", "PlanetTodFitTable", "PointingResidualTable", "SkyCoverageTable", @@ -39,5 +40,6 @@ PipelineInformationTable, SkyCoverageTable, PlanetMapTable, + PlanetMapFitTable, PlanetTodFitTable, ] diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index ae933f2..38c7c32 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -3,12 +3,12 @@ """ from datetime import datetime -from typing import Any +from typing import Any, Optional from astropy.time import Time from astropydantic import AstroPydanticTime -from sqlalchemy import JSON, Column -from sqlmodel import Field, SQLModel +from sqlalchemy import JSON, Column, ForeignKeyConstraint, UniqueConstraint +from sqlmodel import Field, SQLModel, Relationship class PlanetMap(SQLModel): @@ -17,7 +17,6 @@ class PlanetMap(SQLModel): freq_channel: str wafer: str ctime: AstroPydanticTime - dtime: datetime source: str hit_path: str | None @@ -82,16 +81,100 @@ class PlanetMap(SQLModel): class PlanetMapTable(SQLModel, table=True): + """ + A planet map. + + Attributes + ---------- + obs_id : str + observation id + telescope : str + Telescope + freq_channel : str + frequency channel of map + wafer : str + wafer slot + source : str + source. eg jupiter + ctime : datetime + unix time of map + hit_path : str + path of hit map + map_path : str + path of map + weight_path : str + path of weight + weighted_map_path : str + path of weighted_map + elevation : float + elevation of telescope + azimuth : float + azimuth of telescope + pwv: float + mean of precipitable water vapor + pwv_std: float + standard deviation of pwv + pwv_p2p: float + peak-to-peak of pwv + pwv_apex: float + mean of pwv taken by apex + pwv_apex_std: float + standard deviation of pwv apex + pwv_apex_p2p: float + peak-to-peak of pwv apex + f_hwp: float + hwp rotation frequency + roll_angle: float + roll angle + scan_speed: float + scan speed in deg/s + scan_acc: float + scan acceleration in deg/s^2 + sun_distance: float + distance to sun + moon_distance: float + distance to moon + wind_speed: float + wind speed + wind_direction: float + wind direction + ambient_temperature: float + ambient temperature + uv: float + uv index + detnum_before_fitselection: int + number of detectors before fit selection + total_detnum: int + total number of detectors + recenter: bool + recenter flag + yc: float + center of map in y direction + xc: float + center of map in x direction + Tmap_variance: float + variance of T map in pW + Qmap_variance: float + variance of Q map in pW + Umap_variance: float + variance of U map in pW + proc: dict + preprocessing names + detnum: list + list of detector numbers on each proc + detid: list + list of detector ids + """ + __tablename__ = "planet_map" obs_id: str = Field(primary_key=True) telescope: str = Field(primary_key=True) freq_channel: str = Field(primary_key=True) wafer: str = Field(primary_key=True) - ctime: float = Field(nullable=False, primary_key=True) source: str = Field(primary_key=True) - dtime: datetime = Field() + ctime: datetime = Field(nullable=False) hit_path: str | None = Field() map_path: str | None = Field() weight_path: str | None = Field() @@ -139,28 +222,14 @@ class PlanetMapTable(SQLModel, table=True): default=None, sa_column=Column(JSON, nullable=True), ) - # Main beam fit result - peak: float | None = Field() - amplitude: float | None = Field() - xo: float | None = Field() - yo: float | None = Field() - sigmax: float | None = Field() - sigmay: float | None = Field() - theta: float | None = Field() - redchit: float | None = Field() - # Leakage beam fit result - mq: float | None = Field() - d0q: float | None = Field() - d1q: float | None = Field() - sigmaq: float | None = Field() - redchiq: float | None = Field() - mu: float | None = Field() - d0u: float | None = Field() - d1u: float | None = Field() - sigmau: float | None = Field() - redchiu: float | None = Field() - + fit: Optional["PlanetMapFitTable"] = Relationship( + back_populates="map", + cascade_delete=True, + sa_relationship_kwargs={ + "uselist": False, + }, + ) def to_model(self) -> PlanetMap: """ @@ -177,12 +246,11 @@ def to_model(self) -> PlanetMap: freq_channel=self.freq_channel, wafer=self.wafer, ctime=Time(self.ctime, format="unix", scale="utc"), - dtime=self.dtime, source=self.source, hit_path=self.hit_path, map_path=self.map_path, weight_path=self.weight_path, - weighted_map_path=self.weight_map_path, + weighted_map_path=self.weighted_map_path, azimuth=self.azimuth, elevation=self.elevation, duration=self.duration, @@ -213,22 +281,91 @@ def to_model(self) -> PlanetMap: proc=self.proc, detnum=self.detnum, detid=self.detid, - peak=self.peak, - amplitude=self.amplitude, - xo=self.xo, - yo=self.yo, - sigmax=self.sigmax, - sigmay=self.sigmay, - theta=self.theta, - redchit=self.redchit, - mq=self.mq, - d0q=self.d0q, - d1q=self.d1q, - sigmaq=self.sigmaq, - redchiq=self.redchiq, - mu=self.mu, - d0u=self.d0u, - d1u=self.d1u, - sigmau=self.sigmau, - redchiu=self.redchiu, + amplitude=self.fit.amplitude, + xo=self.fit.xo, + yo=self.fit.yo, + sigmax=self.fit.sigmax, + sigmay=self.fit.sigmay, + theta=self.fit.theta, + redchit=self.fit.redchit, + mq=self.fit.mq, + d0q=self.fit.d0q, + d1q=self.fit.d0u, + sigmaq=self.fit.sigmaq, + redchiq=self.fit.redchiq, + mu=self.fit.mu, + d0u=self.fit.d0u, + d1u=self.fit.d1u, + sigmau=self.fit.sigmau, + redchiu=self.fit.redchiu ) + +class PlanetMapFitTable(SQLModel, table=True): + """ + Table for Main-beam and leakage-beam fit results for a planet map. + + Attributes + ---------- + map_id : int + Internal ID of the depth one map + residual_model: ConstantPointingModel | PolynomialPointingModel + The pointing model to actually store in the database. + residual_stats: PointingModelStats + Statistics about the pointing residuals, such as mean and stddev of RA and Dec offsets + """ + + __tablename__ = "planet_map_fits" + __table_args__ = ( + ForeignKeyConstraint( + [ + "obs_id", + "telescope", + "freq_channel", + "wafer", + "source", + ], + [ + "planet_map.obs_id", + "planet_map.telescope", + "planet_map.freq_channel", + "planet_map.wafer", + "planet_map.source", + ], + ondelete="CASCADE", + ), + ) + + # Same primary key as PlanetMapTable + obs_id: str = Field(primary_key=True) + telescope: str = Field(primary_key=True) + freq_channel: str = Field(primary_key=True) + wafer: str = Field(primary_key=True) + source: str = Field(primary_key=True) + + # Main-beam fit result (T map) + peak: float | None = Field() + amplitude: float | None = Field() + xo: float | None = Field() + yo: float | None = Field() + sigmax: float | None = Field() + sigmay: float | None = Field() + theta: float | None = Field() + redchit: float | None = Field() + + # Leakage-beam fit result (Q map) + mq: float | None = Field() + d0q: float | None = Field() + d1q: float | None = Field() + sigmaq: float | None = Field() + redchiq: float | None = Field() + + # Leakage-beam fit result (U map) + mu: float | None = Field() + d0u: float | None = Field() + d1u: float | None = Field() + sigmau: float | None = Field() + redchiu: float | None = Field() + + map: "PlanetMapTable" = Relationship( + back_populates="fit", + ) diff --git a/mapcat/database/planet_todfit.py b/mapcat/database/planet_todfit.py index 4e9a26b..d0f96cd 100644 --- a/mapcat/database/planet_todfit.py +++ b/mapcat/database/planet_todfit.py @@ -15,7 +15,6 @@ class PlanetTodFit(SQLModel): freq_channel: str wafer: str ctime: AstroPydanticTime - dtime: datetime source: str detid: str @@ -49,11 +48,10 @@ class PlanetTodFitTable(SQLModel, table=True): telescope: str = Field(primary_key=True) freq_channel: str = Field(primary_key=True) wafer: str = Field(primary_key=True) - ctime: float = Field(nullable=False, primary_key=True) source: str = Field(primary_key=True) detid: str = Field(primary_key=True) - dtime: datetime = Field() + ctime: datetime = Field(nullable=False) amplitude: float | None = Field() xo: float | None = Field() yo: float | None = Field() @@ -92,7 +90,6 @@ def to_model(self) -> PlanetTodFit: freq_channel=self.freq_channel, wafer=self.wafer, ctime=Time(self.ctime, format="unix", scale="utc"), - dtime=self.dtime, source=self.source, detid=self.detid, amplitude=self.amplitude, From 51e31df3610f17e631c8921f5515f05c5a834321 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Mon, 28 Sep 2026 21:27:03 -0400 Subject: [PATCH 07/18] add docstring --- mapcat/database/planet_todfit.py | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/mapcat/database/planet_todfit.py b/mapcat/database/planet_todfit.py index d0f96cd..1c0149c 100644 --- a/mapcat/database/planet_todfit.py +++ b/mapcat/database/planet_todfit.py @@ -42,6 +42,68 @@ class PlanetTodFit(SQLModel): class PlanetTodFitTable(SQLModel, table=True): + """ + Table for a todfit result of planet observation. + Attributes + ---------- + obs_id : str + observation id + telescope : str + Telescope + freq_channel : str + Frequency band + wafer : str + Wafer slot + source : str + source. eg jupiter + detid : str + detector id + ctime : datetime + start unix time of map + amplitude : float + amplitude of planet signal normalized by solid angle with an unit of degree + xo : float + x offset of planet signal. Unit in degree + yo : float + y offset of planet signal. Unit in degree + sigmax : float + sigma of planet signal in x direction. Unit in degree + sigmay : float + sigma of planet signal in y direction. Unit in degree + theta : float + angle of elliptical gaussian. Unit in degree + defla : float + deflection amplitude. Unit in radian + deflp : float + deflection phase. Unit in radian + amplitude_err : float + error of amplitude + xo_err : float + error of xo + yo_err : float + error of yo + sigmax_err : float + error of sigmax + sigmay_err : float + error of sigmay + theta_err : float + error of theta + defla_err : float + error of defla + deflp_err : float + error of deflp + chisq : float + chi square + dof : int + degree of freedom + xi : float + detector xi position in focal plane coordinate. Unit in radian + eta : float + detector eta position in focal plane coordinate. Unit in radian + gamma : float + detector gamma angle in focal plane coordinate. Unit in radian + + """ __tablename__ = "planet_todfit" obs_id: str = Field(primary_key=True) From 06e0caad7b1d7fd80cc53636d9d825b61db0ac23 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Mon, 28 Sep 2026 21:28:10 -0400 Subject: [PATCH 08/18] fix a bug --- mapcat/database/planet_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index 38c7c32..6d8721c 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -7,7 +7,7 @@ from astropy.time import Time from astropydantic import AstroPydanticTime -from sqlalchemy import JSON, Column, ForeignKeyConstraint, UniqueConstraint +from sqlalchemy import JSON, Column, ForeignKeyConstraint from sqlmodel import Field, SQLModel, Relationship From 471ed450373e757c63d689c2d1be3cae12454732 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Mon, 28 Sep 2026 21:31:36 -0400 Subject: [PATCH 09/18] sort based on ruff --- mapcat/database/__init__.py | 4 ++-- mapcat/database/planet_map.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index a09e5c4..4b37801 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -7,7 +7,7 @@ from .depth_one_coadd import DepthOneCoaddTable from .depth_one_map import DepthOneMapTable from .pipeline_information import PipelineInformationTable -from .planet_map import PlanetMapTable, PlanetMapFitTable +from .planet_map import PlanetMapFitTable, PlanetMapTable from .planet_todfit import PlanetTodFitTable from .pointing_residual import PointingResidualTable from .sky_coverage import SkyCoverageTable @@ -20,8 +20,8 @@ "DepthOneCoaddTable", "DepthOneMapTable", "PipelineInformationTable", - "PlanetMapTable", "PlanetMapFitTable", + "PlanetMapTable", "PlanetTodFitTable", "PointingResidualTable", "SkyCoverageTable", diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index 6d8721c..fdb9da9 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -8,7 +8,7 @@ from astropy.time import Time from astropydantic import AstroPydanticTime from sqlalchemy import JSON, Column, ForeignKeyConstraint -from sqlmodel import Field, SQLModel, Relationship +from sqlmodel import Field, Relationship, SQLModel class PlanetMap(SQLModel): From 6b44c455ad58da0c59f19c4cff3e017e3cadd844 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Tue, 6 Oct 2026 09:47:48 -0400 Subject: [PATCH 10/18] remove unwanted args and add test --- mapcat/database/planet_map.py | 37 --------- tests/test_planet_mapmaking.py | 136 +++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 tests/test_planet_mapmaking.py diff --git a/mapcat/database/planet_map.py b/mapcat/database/planet_map.py index fdb9da9..71ea4bc 100644 --- a/mapcat/database/planet_map.py +++ b/mapcat/database/planet_map.py @@ -58,26 +58,6 @@ class PlanetMap(SQLModel): proc: dict[str, Any] | list[Any] | None = None detnum: list[int] | None = None detid: list[str] | None = None - # Main beam fit result - amplitude: float | None - peak: float | None - xo: float | None - yo: float | None - sigmax: float | None - sigmay: float | None - theta: float | None - redchit: float | None - # Leakage beam fit result - mq: float | None - d0q: float | None - d1q: float | None - sigmaq: float | None - redchiq: float | None - mu: float | None - d0u: float | None - d1u: float | None - sigmau: float | None - redchiu: float | None class PlanetMapTable(SQLModel, table=True): @@ -281,23 +261,6 @@ def to_model(self) -> PlanetMap: proc=self.proc, detnum=self.detnum, detid=self.detid, - amplitude=self.fit.amplitude, - xo=self.fit.xo, - yo=self.fit.yo, - sigmax=self.fit.sigmax, - sigmay=self.fit.sigmay, - theta=self.fit.theta, - redchit=self.fit.redchit, - mq=self.fit.mq, - d0q=self.fit.d0q, - d1q=self.fit.d0u, - sigmaq=self.fit.sigmaq, - redchiq=self.fit.redchiq, - mu=self.fit.mu, - d0u=self.fit.d0u, - d1u=self.fit.d1u, - sigmau=self.fit.sigmau, - redchiu=self.fit.redchiu ) class PlanetMapFitTable(SQLModel, table=True): diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py new file mode 100644 index 0000000..83a2085 --- /dev/null +++ b/tests/test_planet_mapmaking.py @@ -0,0 +1,136 @@ +from datetime import datetime, timezone + +from mapcat.database import PlanetMapTable, PlanetMapFitTable, PlanetTodFitTable +from mapcat.toolkit.mapmaking import build_obslists + + +def test_build_obslists(database_sessionmaker): + # Make some planet maps + with database_sessionmaker() as session: + data1 = PlanetMapTable( + obs_id="obs_1736469509_satp3_1111111", + telescope="satp3", + freq_channel="f150", + wafer="ws0", + source='jupiter', + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + ) + data1.hit_path = "/path/to/hit/file1" + data1.map_path = "/path/to/hit/file2" + data1.proc = ['hoge1', 'hoge2'] + data1.detnum = [10, 20] + data1.detid = ['det1', 'det2'] + mapfit1 = PlanetMapFitTable() + mapfit1.amplitude = 1 + mapfit1.xo = 0 + mapfit1.yo = 0 + mapfit1.sigmax = 1 + mapfit1.sigmay = 1 + mapfit1.theta = 0 + mapfit1.redchit = 1 + mapfit1.mq = 0.1 + mapfit1.d0q = 0.1 + mapfit1.d1q = 0.1 + mapfit1.sigmaq = 0.1 + mapfit1.redchiq = 1 + mapfit1.mu = 0.2 + mapfit1.d0u = 0.2 + mapfit1.d1u = 0.2 + mapfit1.sigmau = 0.2 + mapfit1.redchiu = 1.1 + + data1.fit = mapfit1 + + + data2 = PlanetMapTable( + obs_id="obs_1736469509_satp1_1111111", + telescope="satp1", + freq_channel="f090", + wafer="ws1", + source='saturn', + ctime=datetime.fromtimestamp(1755787530.0, tz=timezone.utc), + ) + data2.proc = ['hoge12', 'hoge2'] + data2.detnum = [12, 22] + data2.detid = ['det12', 'det22'] + + mapfit2 = PlanetMapFitTable() + mapfit2.amplitude = 1 + mapfit2.xo = 0 + mapfit2.yo = 0 + mapfit2.sigmax = 1 + mapfit2.sigmay = 1 + mapfit2.theta = 0 + mapfit2.redchit = 1 + mapfit2.mq = 0.2 + mapfit2.d0q = 0.2 + mapfit2.d1q = 0.2 + mapfit2.sigmaq = 0.2 + mapfit2.redchiq = 2 + mapfit2.mu = 0.3 + mapfit2.d0u = 0.3 + mapfit2.d1u = 0.3 + mapfit2.sigmau = 0.3 + mapfit2.redchiu = 3 + + data2.fit = mapfit2 + + session.add(data1) + session.commit() + session.refresh(data1) + + session.add(data2) + session.commit() + session.refresh(data2) + + obsid1 = data1.obs_id + obsid2 = data2.obs_id + + # Get depth one map back + with database_sessionmaker() as session: + data1 = session.get(PlanetMapTable, data1.obs_id,) + data2 = session.get(PlanetMapTable, data2.obs_id,) + + with database_sessionmaker() as session: + x1 = session.get(PlanetMapTable, obsid1) + session.delete(x1) + session.commit() + + + with database_sessionmaker() as session: + todfit1 = PlanetTodFitTable( + obs_id="obs_1736469509_satp3_1111111", + telescope="satp3", + freq_channel="f150", + wafer="ws0", + source='jupiter', + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + ) + + todfit2 = PlanetTodFitTable( + obs_id="obs_1736469509_satp1_1111111", + telescope="satp1", + freq_channel="f090", + wafer="ws1", + source='saturn', + ctime=datetime.fromtimestamp(1755787526.0, tz=timezone.utc), + ) + + session.add(todfit1) + session.commit() + session.refresh(todfit1) + + session.add(todfit2) + session.commit() + session.refresh(todfit2) + + obsid1 = todfit1.obs_id + + with database_sessionmaker() as session: + y1 = session.get(PlanetTodFitTable, obsid1) + session.delete(y1) + session.commit() + +if __name__ == "__main__": + from sotodlib.database import database_sessionmaker + test_build_obslists(database_sessionmaker) \ No newline at end of file From 794ec339e7644ae4de17611d69a59a07813f3eef Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Tue, 6 Oct 2026 09:50:42 -0400 Subject: [PATCH 11/18] fix a ruff bax --- tests/test_planet_mapmaking.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index 83a2085..021713c 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -1,7 +1,6 @@ from datetime import datetime, timezone from mapcat.database import PlanetMapTable, PlanetMapFitTable, PlanetTodFitTable -from mapcat.toolkit.mapmaking import build_obslists def test_build_obslists(database_sessionmaker): From 3a60e324c50496c9af20e6e57d74e32c4fc5fa9d Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Tue, 6 Oct 2026 09:52:35 -0400 Subject: [PATCH 12/18] fix a ruff bug --- tests/test_planet_mapmaking.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index 021713c..a1d5494 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -83,7 +83,6 @@ def test_build_obslists(database_sessionmaker): session.refresh(data2) obsid1 = data1.obs_id - obsid2 = data2.obs_id # Get depth one map back with database_sessionmaker() as session: @@ -128,8 +127,4 @@ def test_build_obslists(database_sessionmaker): with database_sessionmaker() as session: y1 = session.get(PlanetTodFitTable, obsid1) session.delete(y1) - session.commit() - -if __name__ == "__main__": - from sotodlib.database import database_sessionmaker - test_build_obslists(database_sessionmaker) \ No newline at end of file + session.commit() \ No newline at end of file From 833517b906c758711cf845d7e3ce76205d3d572a Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Tue, 6 Oct 2026 09:57:44 -0400 Subject: [PATCH 13/18] fix a ruff bug --- tests/test_planet_mapmaking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index a1d5494..ce5290f 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone -from mapcat.database import PlanetMapTable, PlanetMapFitTable, PlanetTodFitTable +from mapcat.database import PlanetMapFitTable, PlanetMapTable, PlanetTodFitTable def test_build_obslists(database_sessionmaker): From 1ac2c2fb8c92e301557ddd66acaef2ed926bcb8b Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 7 Oct 2026 11:18:43 -0400 Subject: [PATCH 14/18] wip --- tests/test_planet_mapmaking.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index ce5290f..d3366e3 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -1,6 +1,7 @@ from datetime import datetime, timezone -from mapcat.database import PlanetMapFitTable, PlanetMapTable, PlanetTodFitTable +from mapcat.database import PlanetMapTable, PlanetMapFitTable, PlanetTodFitTable +from mapcat.toolkit.mapmaking import build_obslists def test_build_obslists(database_sessionmaker): @@ -83,6 +84,7 @@ def test_build_obslists(database_sessionmaker): session.refresh(data2) obsid1 = data1.obs_id + obsid2 = data2.obs_id # Get depth one map back with database_sessionmaker() as session: From c53c9e40be3e1dbd1b72dfcf8d316c499fb481bd Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 7 Oct 2026 11:27:39 -0400 Subject: [PATCH 15/18] add alembic --- .../b809eded6421_add_planet_map_tables.py | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py diff --git a/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py new file mode 100644 index 0000000..e4ae183 --- /dev/null +++ b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py @@ -0,0 +1,243 @@ +"""add planet map tables + +Revision ID: b809eded6421 +Revises: 0762b3c7694d +Create Date: 2026-10-07 11:21:13.132297 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b809eded6421' +down_revision: Union[str, None] = '0762b3c7694d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ------------------------------------------------------------------ + # planet_map + # ------------------------------------------------------------------ + op.create_table( + "planet_map", + sa.Column("obs_id", sa.String(), nullable=False), + sa.Column("telescope", sa.String(), nullable=False), + sa.Column("freq_channel", sa.String(), nullable=False), + sa.Column("wafer", sa.String(), nullable=False), + sa.Column("source", sa.String(), nullable=False), + + sa.Column("ctime", sa.DateTime(), nullable=False), + + sa.Column("hit_path", sa.String(), nullable=True), + sa.Column("map_path", sa.String(), nullable=True), + sa.Column("weight_path", sa.String(), nullable=True), + sa.Column("weighted_map_path", sa.String(), nullable=True), + + sa.Column("elevation", sa.Float(), nullable=True), + sa.Column("duration", sa.Float(), nullable=True), + sa.Column("azimuth", sa.Float(), nullable=True), + + sa.Column("pwv", sa.Float(), nullable=True), + sa.Column("pwv_std", sa.Float(), nullable=True), + sa.Column("pwv_p2p", sa.Float(), nullable=True), + + sa.Column("pwv_apex", sa.Float(), nullable=True), + sa.Column("pwv_apex_std", sa.Float(), nullable=True), + sa.Column("pwv_apex_p2p", sa.Float(), nullable=True), + + sa.Column("f_hwp", sa.Float(), nullable=True), + sa.Column("roll_angle", sa.Float(), nullable=True), + sa.Column("scan_speed", sa.Float(), nullable=True), + sa.Column("scan_acc", sa.Float(), nullable=True), + + sa.Column("sun_distance", sa.Float(), nullable=True), + sa.Column("moon_distance", sa.Float(), nullable=True), + + sa.Column("wind_speed", sa.Float(), nullable=True), + sa.Column("wind_direction", sa.Float(), nullable=True), + sa.Column("ambient_temperature", sa.Float(), nullable=True), + sa.Column("uv", sa.Float(), nullable=True), + + sa.Column( + "detnum_before_fitselection", + sa.Integer(), + nullable=True, + ), + sa.Column( + "total_detnum", + sa.Integer(), + nullable=True, + ), + sa.Column( + "recenter", + sa.Boolean(), + nullable=True, + ), + + sa.Column("yc", sa.Float(), nullable=True), + sa.Column("xc", sa.Float(), nullable=True), + + sa.Column( + "Tmap_variance", + sa.Float(), + nullable=True, + ), + sa.Column( + "Qmap_variance", + sa.Float(), + nullable=True, + ), + sa.Column( + "Umap_variance", + sa.Float(), + nullable=True, + ), + + sa.Column( + "proc", + sa.JSON(), + nullable=True, + ), + sa.Column( + "detnum", + sa.JSON(), + nullable=True, + ), + sa.Column( + "detid", + sa.JSON(), + nullable=True, + ), + + sa.PrimaryKeyConstraint( + "obs_id", + "telescope", + "freq_channel", + "wafer", + "source", + ), + ) + + # ------------------------------------------------------------------ + # planet_map_fits + # ------------------------------------------------------------------ + op.create_table( + "planet_map_fits", + sa.Column("obs_id", sa.String(), nullable=False), + sa.Column("telescope", sa.String(), nullable=False), + sa.Column("freq_channel", sa.String(), nullable=False), + sa.Column("wafer", sa.String(), nullable=False), + sa.Column("source", sa.String(), nullable=False), + + # Main-beam fit result (T map) + sa.Column("peak", sa.Float(), nullable=True), + sa.Column("amplitude", sa.Float(), nullable=True), + sa.Column("xo", sa.Float(), nullable=True), + sa.Column("yo", sa.Float(), nullable=True), + sa.Column("sigmax", sa.Float(), nullable=True), + sa.Column("sigmay", sa.Float(), nullable=True), + sa.Column("theta", sa.Float(), nullable=True), + sa.Column("redchit", sa.Float(), nullable=True), + + # Leakage-beam fit result (Q map) + sa.Column("mq", sa.Float(), nullable=True), + sa.Column("d0q", sa.Float(), nullable=True), + sa.Column("d1q", sa.Float(), nullable=True), + sa.Column("sigmaq", sa.Float(), nullable=True), + sa.Column("redchiq", sa.Float(), nullable=True), + + # Leakage-beam fit result (U map) + sa.Column("mu", sa.Float(), nullable=True), + sa.Column("d0u", sa.Float(), nullable=True), + sa.Column("d1u", sa.Float(), nullable=True), + sa.Column("sigmau", sa.Float(), nullable=True), + sa.Column("redchiu", sa.Float(), nullable=True), + + sa.ForeignKeyConstraint( + [ + "obs_id", + "telescope", + "freq_channel", + "wafer", + "source", + ], + [ + "planet_map.obs_id", + "planet_map.telescope", + "planet_map.freq_channel", + "planet_map.wafer", + "planet_map.source", + ], + ondelete="CASCADE", + ), + + sa.PrimaryKeyConstraint( + "obs_id", + "telescope", + "freq_channel", + "wafer", + "source", + ), + ) + + # ------------------------------------------------------------------ + # planet_todfit + # ------------------------------------------------------------------ + op.create_table( + "planet_todfit", + sa.Column("obs_id", sa.String(), nullable=False), + sa.Column("telescope", sa.String(), nullable=False), + sa.Column("freq_channel", sa.String(), nullable=False), + sa.Column("wafer", sa.String(), nullable=False), + sa.Column("source", sa.String(), nullable=False), + sa.Column("detid", sa.String(), nullable=False), + + sa.Column("ctime", sa.DateTime(), nullable=False), + + sa.Column("amplitude", sa.Float(), nullable=True), + sa.Column("xo", sa.Float(), nullable=True), + sa.Column("yo", sa.Float(), nullable=True), + sa.Column("sigmax", sa.Float(), nullable=True), + sa.Column("sigmay", sa.Float(), nullable=True), + sa.Column("theta", sa.Float(), nullable=True), + + sa.Column("defla", sa.Float(), nullable=True), + sa.Column("deflp", sa.Float(), nullable=True), + + sa.Column("amplitude_err", sa.Float(), nullable=True), + sa.Column("xo_err", sa.Float(), nullable=True), + sa.Column("yo_err", sa.Float(), nullable=True), + sa.Column("sigmax_err", sa.Float(), nullable=True), + sa.Column("sigmay_err", sa.Float(), nullable=True), + sa.Column("theta_err", sa.Float(), nullable=True), + + sa.Column("defla_err", sa.Float(), nullable=True), + sa.Column("deflp_err", sa.Float(), nullable=True), + + sa.Column("chisq", sa.Float(), nullable=True), + sa.Column("dof", sa.Integer(), nullable=True), + + sa.Column("xi", sa.Float(), nullable=True), + sa.Column("eta", sa.Float(), nullable=True), + sa.Column("gamma", sa.Float(), nullable=True), + + sa.PrimaryKeyConstraint( + "obs_id", + "telescope", + "freq_channel", + "wafer", + "source", + "detid", + ), + ) + + + +def downgrade() -> None: + op.drop_table("planet_map_fits") + op.drop_table("planet_todfit") + op.drop_table("planet_map") \ No newline at end of file From 703b47eb5c0677b3d97e0bdf656cf2ba15342762 Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 7 Oct 2026 11:47:46 -0400 Subject: [PATCH 16/18] fix to pass test --- .../b809eded6421_add_planet_map_tables.py | 3 +- tests/test_planet_mapmaking.py | 87 +++++++++++++++---- 2 files changed, 73 insertions(+), 17 deletions(-) diff --git a/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py index e4ae183..f30c4c8 100644 --- a/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py +++ b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py @@ -7,9 +7,8 @@ """ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision: str = 'b809eded6421' diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index d3366e3..bb9e68b 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -1,7 +1,6 @@ from datetime import datetime, timezone -from mapcat.database import PlanetMapTable, PlanetMapFitTable, PlanetTodFitTable -from mapcat.toolkit.mapmaking import build_obslists +from mapcat.database import PlanetMapFitTable, PlanetMapTable, PlanetTodFitTable def test_build_obslists(database_sessionmaker): @@ -83,18 +82,31 @@ def test_build_obslists(database_sessionmaker): session.commit() session.refresh(data2) - obsid1 = data1.obs_id - obsid2 = data2.obs_id - + key1 = { + "obs_id": data1.obs_id, + "telescope": data1.telescope, + "freq_channel": data1.freq_channel, + "wafer": data1.wafer, + "source": data1.source, + } + + key2 = { + "obs_id": data2.obs_id, + "telescope": data2.telescope, + "freq_channel": data2.freq_channel, + "wafer": data2.wafer, + "source": data2.source, + } # Get depth one map back with database_sessionmaker() as session: - data1 = session.get(PlanetMapTable, data1.obs_id,) - data2 = session.get(PlanetMapTable, data2.obs_id,) + data1 = session.get(PlanetMapTable, key1) + data2 = session.get(PlanetMapTable, key2) with database_sessionmaker() as session: - x1 = session.get(PlanetMapTable, obsid1) + x1 = session.get(PlanetMapTable, key1) session.delete(x1) session.commit() + assert x1 is not None with database_sessionmaker() as session: @@ -103,8 +115,20 @@ def test_build_obslists(database_sessionmaker): telescope="satp3", freq_channel="f150", wafer="ws0", - source='jupiter', - ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + source="jupiter", + detid="det1", + ctime=datetime.fromtimestamp( + 1755787524.0, + tz=timezone.utc, + ), + amplitude=1.0, + xo=0.0, + yo=0.0, + sigmax=1.0, + sigmay=1.0, + theta=0.0, + chisq=1.0, + dof=100, ) todfit2 = PlanetTodFitTable( @@ -112,8 +136,20 @@ def test_build_obslists(database_sessionmaker): telescope="satp1", freq_channel="f090", wafer="ws1", - source='saturn', - ctime=datetime.fromtimestamp(1755787526.0, tz=timezone.utc), + source="saturn", + detid="det12", + ctime=datetime.fromtimestamp( + 1755787526.0, + tz=timezone.utc, + ), + amplitude=1.1, + xo=0.1, + yo=0.1, + sigmax=1.1, + sigmay=1.1, + theta=0.1, + chisq=1.1, + dof=101, ) session.add(todfit1) @@ -124,9 +160,30 @@ def test_build_obslists(database_sessionmaker): session.commit() session.refresh(todfit2) - obsid1 = todfit1.obs_id + key_tod1 = { + "obs_id": todfit1.obs_id, + "telescope": todfit1.telescope, + "freq_channel": todfit1.freq_channel, + "wafer": todfit1.wafer, + "source": todfit1.source, + "detid": todfit1.detid, + } + key_tod2 = { + "obs_id": todfit2.obs_id, + "telescope": todfit2.telescope, + "freq_channel": todfit2.freq_channel, + "wafer": todfit2.wafer, + "source": todfit2.source, + "detid": todfit2.detid, + } + + + with database_sessionmaker() as session: + y1 = session.get(PlanetTodFitTable, key_tod1) + y2 = session.get(PlanetTodFitTable, key_tod2) with database_sessionmaker() as session: - y1 = session.get(PlanetTodFitTable, obsid1) + y1 = session.get(PlanetTodFitTable, key_tod1) session.delete(y1) - session.commit() \ No newline at end of file + session.commit() + assert y1 is not None \ No newline at end of file From d1e83263901f8b6a44a66201b751cf4f682d040e Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 7 Oct 2026 11:54:43 -0400 Subject: [PATCH 17/18] fix a ruff error --- .../versions/b809eded6421_add_planet_map_tables.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py index f30c4c8..2887c80 100644 --- a/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py +++ b/mapcat/alembic/versions/b809eded6421_add_planet_map_tables.py @@ -5,16 +5,16 @@ Create Date: 2026-10-07 11:21:13.132297 """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = 'b809eded6421' -down_revision: Union[str, None] = '0762b3c7694d' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = '0762b3c7694d' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: From ec5ffe08dc24824a49f36235f67127203b1237fc Mon Sep 17 00:00:00 2001 From: Yoshinori Sueno Date: Wed, 7 Oct 2026 11:59:31 -0400 Subject: [PATCH 18/18] remove y2 for the test --- tests/test_planet_mapmaking.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_planet_mapmaking.py b/tests/test_planet_mapmaking.py index bb9e68b..ab48258 100644 --- a/tests/test_planet_mapmaking.py +++ b/tests/test_planet_mapmaking.py @@ -168,19 +168,9 @@ def test_build_obslists(database_sessionmaker): "source": todfit1.source, "detid": todfit1.detid, } - key_tod2 = { - "obs_id": todfit2.obs_id, - "telescope": todfit2.telescope, - "freq_channel": todfit2.freq_channel, - "wafer": todfit2.wafer, - "source": todfit2.source, - "detid": todfit2.detid, - } - with database_sessionmaker() as session: y1 = session.get(PlanetTodFitTable, key_tod1) - y2 = session.get(PlanetTodFitTable, key_tod2) with database_sessionmaker() as session: y1 = session.get(PlanetTodFitTable, key_tod1)