From 8347e35e12db3a61bdf99cb28ab0b9fa056f897c Mon Sep 17 00:00:00 2001 From: Russ Biggs Date: Wed, 28 Aug 2024 08:01:05 -0600 Subject: [PATCH] Feature/latest endpoints (#372) * add latest endpoints and test * fix datetime_from datetime_to date_from and date_to docstring about tz * Added some tests --------- Co-authored-by: Christian Parker --- openaq_api/openaq_api/main.py | 13 +- openaq_api/openaq_api/v3/models/queries.py | 16 +- openaq_api/openaq_api/v3/models/responses.py | 24 ++- openaq_api/openaq_api/v3/routers/latest.py | 175 +++++++++++++++++++ openaq_api/openaq_api/v3/routers/sensors.py | 4 +- openaq_api/tests/test_sensors_latest.py | 68 +++++++ openaq_api/tests/unit/test_v3_queries.py | 57 ++++++ 7 files changed, 336 insertions(+), 21 deletions(-) create mode 100644 openaq_api/openaq_api/v3/routers/latest.py create mode 100644 openaq_api/tests/test_sensors_latest.py diff --git a/openaq_api/openaq_api/main.py b/openaq_api/openaq_api/main.py index f3a4208..0ce873e 100644 --- a/openaq_api/openaq_api/main.py +++ b/openaq_api/openaq_api/main.py @@ -32,7 +32,8 @@ UnprocessableEntityLog, WarnLog, ) -#from openaq_api.routers.auth import router as auth_router + +# from openaq_api.routers.auth import router as auth_router from openaq_api.routers.averages import router as averages_router from openaq_api.routers.cities import router as cities_router from openaq_api.routers.countries import router as countries_router @@ -61,6 +62,7 @@ tiles, trends, licenses, + latest, ) logging.basicConfig( @@ -193,13 +195,12 @@ class OpenAQValidationResponse(BaseModel): detail: list[OpenAQValidationResponseDetail] | None = None - @app.exception_handler(RequestValidationError) async def openaq_request_validation_exception_handler( request: Request, exc: RequestValidationError ): return ORJSONResponse(status_code=422, content=jsonable_encoder(str(exc))) - #return PlainTextResponse(str(exc)) + # return PlainTextResponse(str(exc)) # print("\n\n\n\n\n") # print(str(exc)) # print("\n\n\n\n\n") @@ -209,7 +210,7 @@ async def openaq_request_validation_exception_handler( # UnprocessableEntityLog(request=request, detail=str(exc)).model_dump_json() # ) # detail = OpenAQValidationResponse(detail=detail) - #return ORJSONResponse(status_code=422, content=jsonable_encoder(detail)) + # return ORJSONResponse(status_code=422, content=jsonable_encoder(detail)) @app.exception_handler(ValidationError) @@ -222,7 +223,7 @@ async def openaq_exception_handler(request: Request, exc: ValidationError): # request=request, detail=exc.jsmodel_dump_jsonon() # ).model_dump_json() # ) - #return ORJSONResponse(status_code=422, content=jsonable_encoder(detail)) + # return ORJSONResponse(status_code=422, content=jsonable_encoder(detail)) # return ORJSONResponse(status_code=500, content={"message": "internal server error"}) @@ -256,6 +257,8 @@ def favico(): app.include_router(trends.router) app.include_router(providers.router) app.include_router(sensors.router) +app.include_router(latest.router) + # app.include_router(auth_router) app.include_router(averages_router) diff --git a/openaq_api/openaq_api/v3/models/queries.py b/openaq_api/openaq_api/v3/models/queries.py index db7f34e..3bbae76 100644 --- a/openaq_api/openaq_api/v3/models/queries.py +++ b/openaq_api/openaq_api/v3/models/queries.py @@ -506,7 +506,7 @@ def where(self) -> str: Overrides the base QueryBaseModel `where` method If `datetime_from` is a `date` or `datetime` without a timezone a timezone - is added as UTC. + is added as local timezone. Returns: string of WHERE clause if `datetime_from` is set @@ -544,13 +544,13 @@ class DatetimeToQuery(QueryBaseModel): def where(self) -> str: """Generates SQL condition for filtering to datetime. - Overrides the base QueryBaseModel `where` method + Overrides the base QueryBaseModel `where` method - If `datetime_to` is a `date` or `datetime` without a timezone a timezone - is added as UTC. + If `datetime_to` is a `date` or `datetime` without a timezone a timezone + is added as local timezone. - Returns: - string of WHERE clause if `datetime_to` is set + Returns: + string of WHERE clause if `datetime_to` is set """ tz = self.map("timezone", "timezone") dt = self.map("datetime", "datetime") @@ -587,7 +587,7 @@ def where(self) -> str: Overrides the base QueryBaseModel `where` method If `date_from` is a `date` or `datetime` without a timezone a timezone - is added as UTC. + is added as local timezone. Returns: string of WHERE clause if `date_from` is set @@ -624,7 +624,7 @@ def where(self) -> str: Overrides the base QueryBaseModel `where` method If `date_to` is a `date` or `datetime` without a timezone a timezone - is added as UTC. + is added as local timezone. Returns: string of WHERE clause if `date_to` is set diff --git a/openaq_api/openaq_api/v3/models/responses.py b/openaq_api/openaq_api/v3/models/responses.py index cd28206..713a069 100644 --- a/openaq_api/openaq_api/v3/models/responses.py +++ b/openaq_api/openaq_api/v3/models/responses.py @@ -142,7 +142,7 @@ class License(JsonBase): source_url: str -class Latest(JsonBase): +class LatestBase(JsonBase): datetime: DatetimeObject value: float coordinates: Coordinates @@ -213,10 +213,15 @@ class Sensor(SensorBase): datetime_first: DatetimeObject | None = None datetime_last: DatetimeObject | None = None coverage: Coverage - latest: Latest + latest: LatestBase summary: Summary +class Latest(LatestBase): + sensors_id: int + locations_id: int + + class Location(JsonBase): id: int name: str | None = None @@ -248,8 +253,8 @@ class Measurement(JsonBase): class HourlyData(JsonBase): - #datetime: DatetimeObject - value: float | None = None # Nullable to deal with errors + # datetime: DatetimeObject + value: float | None = None # Nullable to deal with errors parameter: ParameterBase period: Period | None = None coordinates: Coordinates | None = None @@ -258,7 +263,7 @@ class HourlyData(JsonBase): class DailyData(JsonBase): - #datetime: DatetimeObject + # datetime: DatetimeObject value: float parameter: ParameterBase period: Period | None = None @@ -268,7 +273,7 @@ class DailyData(JsonBase): class AnnualData(JsonBase): - #datetime: DatetimeObject + # datetime: DatetimeObject value: float parameter: ParameterBase period: Period | None = None @@ -301,12 +306,15 @@ class LocationsResponse(OpenAQResult): class MeasurementsResponse(OpenAQResult): results: list[Measurement] + class HourlyDataResponse(OpenAQResult): results: list[HourlyData] + class DailyDataResponse(OpenAQResult): results: list[DailyData] + class AnnualDataResponse(OpenAQResult): results: list[AnnualData] @@ -341,3 +349,7 @@ class ManufacturersResponse(OpenAQResult): class OwnersResponse(OpenAQResult): results: list[Owner] + + +class LatestResponse(OpenAQResult): + results: list[Latest] diff --git a/openaq_api/openaq_api/v3/routers/latest.py b/openaq_api/openaq_api/v3/routers/latest.py new file mode 100644 index 0000000..aeb4e07 --- /dev/null +++ b/openaq_api/openaq_api/v3/routers/latest.py @@ -0,0 +1,175 @@ +from datetime import date, datetime +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Path, Query + +from openaq_api.db import DB +from openaq_api.v3.models.queries import QueryBaseModel, QueryBuilder, Paging +from openaq_api.v3.models.responses import LatestResponse + +logger = logging.getLogger("latest") + +router = APIRouter( + prefix="/v3", + tags=["v3"], + include_in_schema=True, +) + + +class DatetimeMinQuery(QueryBaseModel): + """Pydantic query model for the `datetime_min` query parameter + + Inherits from QueryBaseModel + + Attributes: + datetime_min: date or datetime in ISO-8601 format to filter results to a mininum data + """ + + datetime_min: datetime | date | None = Query( + None, + description="Minimum datetime", + examples=["2022-10-01T11:19:38-06:00", "2022-10-01"], + ) + + def where(self) -> str: + """Generates SQL condition for filtering to datetime. + + Overrides the base QueryBaseModel `where` method + + If `datetime_min` is a `date` or `datetime` without a timezone a timezone + is added as local timezone. + + Returns: + string of WHERE clause if `datetime_min` is set + """ + tz = self.map("timezone", "tzid") + dt = self.map("datetime", "datetime_last") + + if self.datetime_min is None: + return None + elif isinstance(self.datetime_min, datetime): + if self.datetime_min.tzinfo is None: + return f"{dt} > (:datetime_min::timestamp AT TIME ZONE {tz})" + else: + return f"{dt} > :datetime_min" + elif isinstance(self.datetime_min, date): + return f"{dt} > (:datetime_min::timestamp AT TIME ZONE {tz})" + + +class ParameterLatestPathQuery(QueryBaseModel): + """Path query to filter results by parameters ID + + Inherits from QueryBaseModel + + Attributes: + parameters_id: countries ID value + """ + + parameters_id: int = Path( + ..., description="Limit the results to a specific parameters id", ge=1 + ) + + def where(self) -> str: + """Generates SQL condition for filtering to a single parameters_id + + Overrides the base QueryBaseModel `where` method + + Returns: + string of WHERE clause + """ + return "m.measurands_id = :parameters_id" + + +class ParametersLatestQueries(ParameterLatestPathQuery, DatetimeMinQuery, Paging): ... + + +@router.get( + "/parameters/{parameters_id}/latest", + response_model=LatestResponse, + summary="Get a owner by ID", + description="Provides a owner by owner ID", +) +async def parameters_latest_get( + parameters_latest: Annotated[ + ParametersLatestQueries, Depends(ParametersLatestQueries.depends()) + ], + db: DB = Depends(), +): + response = await fetch_latest(parameters_latest, db) + return response + + +class LocationLatestPathQuery(QueryBaseModel): + """Path query to filter results by locations ID. + + Inherits from QueryBaseModel. + + Attributes: + locations_id: locations ID value. + """ + + locations_id: int = Path( + description="Limit the results to a specific location by id", ge=1 + ) + + def where(self) -> str: + """Generates SQL condition for filtering to a single locations_id + + Overrides the base QueryBaseModel `where` method + + Returns: + string of WHERE clause + """ + return "n.sensor_nodes_id = :locations_id" + + +class LocationsLatestQueries(LocationLatestPathQuery, DatetimeMinQuery, Paging): ... + + +@router.get( + "/locations/{locations_id}/latest", + response_model=LatestResponse, + summary="Get a owner by ID", + description="Provides a owner by owner ID", +) +async def owner_get( + locations_latest: Annotated[ + LocationsLatestQueries, Depends(LocationsLatestQueries.depends()) + ], + db: DB = Depends(), +): + response = await fetch_latest(locations_latest, db) + return response + + +async def fetch_latest(query, db): + query_builder = QueryBuilder(query) + sql = f""" + SELECT + n.sensor_nodes_id AS locations_id + ,s.sensors_id AS sensors_id + ,get_datetime_object(r.datetime_last, t.tzid) as datetime + ,r.value_latest AS value + ,json_build_object( + 'latitude', st_y(COALESCE(r.geom_latest, n.geom)) + ,'longitude', st_x(COALESCE(r.geom_latest, n.geom)) + ) AS coordinates + {query_builder.total()} + FROM + sensors s + JOIN + sensor_systems sy ON (s.sensor_systems_id = sy.sensor_systems_id) + JOIN + sensor_nodes n ON (sy.sensor_nodes_id = n.sensor_nodes_id) + JOIN + timezones t ON (n.timezones_id = t.timezones_id) + JOIN + measurands m ON (s.measurands_id = m.measurands_id) + LEFT JOIN + sensors_rollup r ON (s.sensors_id = r.sensors_id) + {query_builder.where()} + {query_builder.pagination()} + """ + response = await db.fetchPage(sql, query_builder.params()) + return response diff --git a/openaq_api/openaq_api/v3/routers/sensors.py b/openaq_api/openaq_api/v3/routers/sensors.py index 07f778e..fb9a6c8 100644 --- a/openaq_api/openaq_api/v3/routers/sensors.py +++ b/openaq_api/openaq_api/v3/routers/sensors.py @@ -22,7 +22,7 @@ ) -class SensorQuery(QueryBaseModel): +class SensorPathQuery(QueryBaseModel): sensors_id: int = Path( ..., description="Limit the results to a specific sensors id", ge=1 ) @@ -62,7 +62,7 @@ async def sensors_get( description="Provides a sensor by sensor ID", ) async def sensor_get( - sensors: Annotated[SensorQuery, Depends(SensorQuery.depends())], + sensors: Annotated[SensorPathQuery, Depends(SensorPathQuery.depends())], db: DB = Depends(), ): return await fetch_sensors(sensors, db) diff --git a/openaq_api/tests/test_sensors_latest.py b/openaq_api/tests/test_sensors_latest.py new file mode 100644 index 0000000..cd312c3 --- /dev/null +++ b/openaq_api/tests/test_sensors_latest.py @@ -0,0 +1,68 @@ +from fastapi.testclient import TestClient +import json +import time +import pytest +from openaq_api.main import app + + +@pytest.fixture +def client(): + with TestClient(app) as c: + yield c + + +measurands_id = 2 +# location 1 is at -10 hrs +# last value is on 2024-08-27 19:30 +locations_id = 1 + +class TestLocations: + def test_default_good(self, client): + response = client.get(f"/v3/locations/{locations_id}/latest") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + + def test_date_filter(self, client): + response = client.get(f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + + def test_timestamp_filter(self, client): + response = client.get(f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27 19:00:00") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + + def test_timestamptz_filter(self, client): + response = client.get(f"/v3/locations/{locations_id}/latest?datetime_min=2024-08-27 19:00:00-10:00") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + + +class TestMeasurands: + def test_default_good(self, client): + response = client.get(f"/v3/parameters/{measurands_id}/latest") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 5 + + def test_date_filter(self, client): + response = client.get(f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 10 + + def test_timestamp_filter(self, client): + response = client.get(f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27 19:00:00") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 + + def test_timestamptz_filter(self, client): + response = client.get(f"/v3/parameters/{measurands_id}/latest?datetime_min=2024-08-27 19:00:00-10:00") + assert response.status_code == 200 + data = json.loads(response.content).get('results', []) + assert len(data) == 1 diff --git a/openaq_api/tests/unit/test_v3_queries.py b/openaq_api/tests/unit/test_v3_queries.py index 42c508e..9bb4f9b 100644 --- a/openaq_api/tests/unit/test_v3_queries.py +++ b/openaq_api/tests/unit/test_v3_queries.py @@ -7,6 +7,11 @@ from buildpg import render from pydantic import TypeAdapter +from openaq_api.v3.routers.latest import ( + ParameterLatestPathQuery, + LocationLatestPathQuery, + DatetimeMinQuery, +) from openaq_api.v3.models.queries import ( BboxQuery, CommaSeparatedList, @@ -604,3 +609,55 @@ def test_no_value(self): params = manufacturers_query.model_dump() assert where is None assert params == {"manufacturers_id": None} + + +class TestParameterLatestPathQuery: + def test_has_value(self): + parameter_latest_path_query = ParameterLatestPathQuery(parameters_id=42) + where = parameter_latest_path_query.where() + params = parameter_latest_path_query.model_dump() + assert where == "m.measurands_id = :parameters_id" + assert params == {"parameters_id": 42} + + def test_no_value(self): + with pytest.raises(fastapi.exceptions.HTTPException): + ParameterLatestPathQuery() + + +class TestLocationLatestPathQuery: + def test_has_value(self): + location_latest_path_query = LocationLatestPathQuery(locations_id=42) + where = location_latest_path_query.where() + params = location_latest_path_query.model_dump() + assert where == "n.sensor_nodes_id = :locations_id" + assert params == {"locations_id": 42} + + def test_no_value(self): + with pytest.raises(fastapi.exceptions.HTTPException): + LocationLatestPathQuery() + + +class TestDatetimeMinQuery: + def test_has_no_value(self): + query = DatetimeMinQuery(datetime_min=None) + params = query.model_dump() + assert params == {"datetime_min": None} + assert query.where() == None + + def test_has_date_value(self): + query = DatetimeMinQuery(datetime_min='2024-01-01') + params = query.model_dump() + assert params == {"datetime_min": date(2024, 1, 1)} + assert query.where() == "datetime_last > (:datetime_min::timestamp AT TIME ZONE tzid)" + + def test_has_timestamp_value(self): + query = DatetimeMinQuery(datetime_min='2024-01-01 01:01:01') + params = query.model_dump() + assert params == {"datetime_min": datetime(2024, 1, 1, 1, 1, 1)} + assert query.where() == "datetime_last > (:datetime_min::timestamp AT TIME ZONE tzid)" + + def test_has_timestamptz_value(self): + query = DatetimeMinQuery(datetime_min='2024-01-01 01:01:01-00:00') + params = query.model_dump() + assert params == {"datetime_min": datetime(2024, 1, 1, 1, 1, 1, tzinfo=ZoneInfo('UTC'))} + assert query.where() == "datetime_last > :datetime_min"