Features/v3 annual averaging (#352)

* Updated to let users aggregate to annual average

  Also added the ability to change the base table from hourly to
  daily, though this has not been implimented yet.

* Added ability to change base table to sensors

* Added new paths and tests and have basic working examples

* add display name for measurand

* revert

* Testing out a method where we can alter query strings
  This allows us to use the same query class even when we just need to
  alter the query field

* Updated tests and queries

  Not passing all tests yet but very close

* One test still failing: days -> moy

  The same query seems to work locally but not in the test?

* Fixed timestamp issues - passing all tests

---------

Co-authored-by: Russ Biggs <[email protected]>

.gitignore

@@ -9,4 +9,5 @@ cdk.out/
 openaq_api/openaq_api/templates/*
 openaq_api/openaq_api/static/assets/*
 .python-version
-Pipfile
+Pipfile
+/openaq_api/venv/

openaq_api/openaq_api/main.py

@@ -193,6 +193,7 @@ class OpenAQValidationResponse(BaseModel):
     detail: list[OpenAQValidationResponseDetail] | None = None
 
 
+
 @app.exception_handler(RequestValidationError)
 async def openaq_request_validation_exception_handler(
     request: Request, exc: RequestValidationError

openaq_api/openaq_api/routers/averages.py

@@ -5,8 +5,8 @@
 from fastapi import APIRouter, Depends, Query
 
 from openaq_api.v3.models.queries import (
-    DateFromQuery,
-    DateToQuery,
+    DatetimeFromQuery,
+    DatetimeToQuery,
     Paging,
     TemporalQuery,
     QueryBaseModel,
@@ -59,8 +59,8 @@ class AveragesQueries(
     Paging,
     SpatialTypeQuery,
     LocationQuery,
-    DateFromQuery,
-    DateToQuery,
+    DatetimeFromQuery,
+    DatetimeToQuery,
     ParametersQuery,
     TemporalQuery,
 ):
@@ -98,8 +98,8 @@ async def averages_v2_get(
         , m.measurands_id as "parameterId"
         , m.display as "displayName"
         , m.units as unit
-        , h.first_datetime
-        , h.last_datetime
+        , h.datetime_first as first_datetime
+        , h.datetime_last as last_datetime
         {query.fields()}
         FROM hourly_data h
         JOIN sensors s USING (sensors_id)

openaq_api/openaq_api/v3/models/queries.py

@@ -216,6 +216,10 @@ def depends(cls):
         """ """
         return parameter_dependency_from_model("depends", cls)
 
+    def map(self, key: str, default: str | None = None):
+        cols = getattr(self, '__column_map__', {})
+        return cols.get(key, default)
+
     def has(self, field_name: str) -> bool:
         """ """
         return hasattr(self, field_name) and getattr(self, field_name) is not None
@@ -460,6 +464,89 @@ def where(self) -> str | None:
             return "country->>'code' = :iso"
 
 
+class DatetimeFromQuery(QueryBaseModel):
+    """Pydantic query model for the `datetime_from` query parameter
+
+    Inherits from QueryBaseModel
+
+    Attributes:
+        datetime_from: date or datetime in ISO-8601 format to filter results to a
+        date range.
+    """
+
+    datetime_from: datetime | date | None = Query(
+        None,
+        description="From when?",
+        examples=["2022-10-01T11:19:38-06:00", "2022-10-01"],
+    )
+
+
+    def where(self) -> str:
+        """Generates SQL condition for filtering to datetime.
+
+        Overrides the base QueryBaseModel `where` method
+
+        If `datetime_from` is a `date` or `datetime` without a timezone a timezone
+        is added as UTC.
+
+        Returns:
+            string of WHERE clause if `datetime_from` is set
+        """
+        tz = self.map('timezone', 'timezone')
+        dt = self.map('datetime', 'datetime')
+
+        if self.datetime_from is None:
+            return None
+        elif isinstance(self.datetime_from, datetime):
+            if self.datetime_from.tzinfo is None:
+                return f"{dt} > (:datetime_from::timestamp AT TIME ZONE {tz})"
+            else:
+                return f"{dt} > :datetime_from"
+        elif isinstance(self.datetime_from, date):
+            return f"{dt} > (:datetime_from::timestamp AT TIME ZONE {tz})"
+
+
+class DatetimeToQuery(QueryBaseModel):
+    """Pydantic query model for the `date_to` query parameter
+
+    Inherits from QueryBaseModel
+
+    Attributes:
+        date_to: date or datetime in ISO-8601 format to filter results to a
+        date range.
+    """
+
+    datetime_to: datetime | date | None = Query(
+        None,
+        description="To when?",
+        examples=["2022-10-01T11:19:38-06:00", "2022-10-01"],
+    )
+
+    def where(self) -> str:
+        """Generates SQL condition for filtering to datetime.
+
+        Overrides the base QueryBaseModel `where` method
+
+        If `datetime_to` is a `date` or `datetime` without a timezone a timezone
+        is added as UTC.
+
+        Returns:
+            string of WHERE clause if `datetime_to` is set
+        """
+        tz = self.map('timezone', 'timezone')
+        dt = self.map('datetime', 'datetime')
+
+        if self.datetime_to is None:
+            return None
+        elif isinstance(self.datetime_to, datetime):
+            if self.datetime_to.tzinfo is None:
+                return f"{dt} <= (:datetime_to::timestamp AT TIME ZONE {tz})"
+            else:
+                return f"{dt} <= :datetime_to"
+        elif isinstance(self.datetime_to, date):
+            return f"{dt} <= (:datetime_to::timestamp AT TIME ZONE {tz})"
+
+
 class DateFromQuery(QueryBaseModel):
     """Pydantic query model for the `date_from` query parameter
 
@@ -476,6 +563,7 @@ class DateFromQuery(QueryBaseModel):
         examples=["2022-10-01T11:19:38-06:00", "2022-10-01"],
     )
 
+
     def where(self) -> str:
         """Generates SQL condition for filtering to datetime.
 
@@ -487,16 +575,14 @@ def where(self) -> str:
         Returns:
             string of WHERE clause if `date_from` is set
         """
+        dt = self.map('datetime', 'datetime')
 
         if self.date_from is None:
             return None
         elif isinstance(self.date_from, datetime):
-            if self.date_from.tzinfo is None:
-                return "datetime > (:date_from::timestamp AT TIME ZONE timezone)"
-            else:
-                return "datetime > :date_from"
+            return f"{dt} >= :date_from::date"
         elif isinstance(self.date_from, date):
-            return "datetime > (:date_from::timestamp AT TIME ZONE timezone)"
+            return f"{dt} >= :date_from::date"
 
 
 class DateToQuery(QueryBaseModel):
@@ -526,15 +612,14 @@ def where(self) -> str:
         Returns:
             string of WHERE clause if `date_to` is set
         """
+        dt = self.map('datetime', 'datetime')
+
         if self.date_to is None:
             return None
         elif isinstance(self.date_to, datetime):
-            if self.date_to.tzinfo is None:
-                return "datetime <= (:date_to::timestamp AT TIME ZONE timezone)"
-            else:
-                return "datetime <= :date_to"
+            return f"{dt} <= :date_to::date"
         elif isinstance(self.date_to, date):
-            return "datetime <= (:date_to::timestamp AT TIME ZONE timezone)"
+            return f"{dt} <= :date_to::date"
 
 
 class PeriodNames(StrEnum):
@@ -548,6 +633,7 @@ class PeriodNames(StrEnum):
     raw = "raw"
 
 
+
 class PeriodNameQuery(QueryBaseModel):
     """Pydantic query model for the `period_name` query parameter.
 
@@ -558,11 +644,11 @@ class PeriodNameQuery(QueryBaseModel):
     """
 
     period_name: PeriodNames | None = Query(
-        "hour",
-        description="Period to aggregate. Month, day, hour, hour of day (hod), day of week (dow) and month of year (moy)",
+        "hour", description="Period to aggregate. Year, month, day, hour, hour of day (hod), day of week (dow) and month of year (moy)"
     )
 
 
+
 class TemporalQuery(QueryBaseModel):
     """Pydantic query model for the `period_name` query parameter.
 
@@ -856,6 +942,13 @@ def _sortable(self) -> SortingBase | None:
         else:
             return None
 
+    def set_column_map(self, m: dict):
+        """
+        Provide a dictionary that can be used later in the where methods
+        to dynamically set a query field name.
+        """
+        setattr(self, '__column_map__', m)
+
     def fields(self) -> str:
         """
         loops through all ancestor classes and calls
@@ -912,8 +1005,10 @@ def where(self) -> str:
         bases = self._bases()
         for base in bases:
             if callable(getattr(base, "where", None)):
-                if base.where(self.query):
-                    where.append(base.where(self.query))
+                setattr(self.query, '__column_map__', getattr(self, '__column_map__', {}))
+                clause = base.where(self.query)
+                if clause:
+                    where.append(clause)
         if len(where):
             where = list(set(where))
             where.sort()  # ensure the order is consistent for testing

openaq_api/openaq_api/v3/models/responses.py

@@ -247,6 +247,36 @@ class Measurement(JsonBase):
     coverage: Coverage | None = None
 
 
+class HourlyData(JsonBase):
+    #datetime: DatetimeObject
+    value: float
+    parameter: ParameterBase
+    period: Period | None = None
+    coordinates: Coordinates | None = None
+    summary: Summary | None = None
+    coverage: Coverage | None = None
+
+
+class DailyData(JsonBase):
+    #datetime: DatetimeObject
+    value: float
+    parameter: ParameterBase
+    period: Period | None = None
+    coordinates: Coordinates | None = None
+    summary: Summary | None = None
+    coverage: Coverage | None = None
+
+
+class AnnualData(JsonBase):
+    #datetime: DatetimeObject
+    value: float
+    parameter: ParameterBase
+    period: Period | None = None
+    coordinates: Coordinates | None = None
+    summary: Summary | None = None
+    coverage: Coverage | None = None
+
+
 # Similar to measurement but without timestamps
 class Trend(JsonBase):
     factor: Factor
@@ -271,6 +301,15 @@ class LocationsResponse(OpenAQResult):
 class MeasurementsResponse(OpenAQResult):
     results: list[Measurement]
 
+class HourlyDataResponse(OpenAQResult):
+    results: list[HourlyData]
+
+class DailyDataResponse(OpenAQResult):
+    results: list[DailyData]
+
+class AnnualDataResponse(OpenAQResult):
+    results: list[AnnualData]
+
 
 class TrendsResponse(OpenAQResult):
     results: list[Trend]

openaq_api/openaq_api/v3/routers/measurements.py

@@ -5,8 +5,8 @@
 from openaq_api.db import DB
 from openaq_api.v3.models.queries import (
     CommaSeparatedList,
-    DateFromQuery,
-    DateToQuery,
+    DatetimeFromQuery,
+    DatetimeToQuery,
     Paging,
     PeriodNameQuery,
     QueryBaseModel,
@@ -41,8 +41,8 @@ def where(self) -> str | None:
 class LocationMeasurementsQueries(
     Paging,
     LocationPathQuery,
-    DateFromQuery,
-    DateToQuery,
+    DatetimeFromQuery,
+    DatetimeToQuery,
     MeasurementsParametersQuery,
     PeriodNameQuery,
 ): ...
@@ -64,6 +64,8 @@ async def measurements_get(
     return response
 
 
+
+
 async def fetch_measurements(q, db):
     query = QueryBuilder(q)
     dur = "01:00:00"
@@ -101,8 +103,8 @@ async def fetch_measurements(q, db):
         , s.data_logging_period_seconds
         , {expected_hours} * 3600
         )||jsonb_build_object(
-          'datetime_from', get_datetime_object(h.first_datetime, sn.timezone)
-        , 'datetime_to', get_datetime_object(h.last_datetime, sn.timezone)
+          'datetime_from', get_datetime_object(h.datetime_first, sn.timezone)
+        , 'datetime_to', get_datetime_object(h.datetime_last, sn.timezone)
         ) as coverage
         {query.fields()}
         FROM hourly_data h
@@ -122,6 +124,8 @@ async def fetch_measurements(q, db):
             dur = "24:00:00"
         elif q.period_name == "month":
             dur = "1 month"
+        elif q.period_name == "year":
+            dur = "1 year"
 
         sql = f"""
             WITH meas AS (