Merge pull request #255 from hotosm/feature/yaml-support

Feature : Custom Exports YAML support

API/custom_exports.py

@@ -1,11 +1,19 @@
+# Standard library imports
+import json
+from typing import Dict
+
+# Third party imports
+import yaml
 from fastapi import APIRouter, Body, Depends, HTTPException, Request
 from fastapi.responses import JSONResponse
 from fastapi_versioning import version
+from pydantic import ValidationError
 
+# Reader imports
 from src.config import DEFAULT_QUEUE_NAME
 from src.config import LIMITER as limiter
 from src.config import RATE_LIMIT_PER_MIN
-from src.validation.models import DynamicCategoriesModel
+from src.validation.models import CustomRequestsYaml, DynamicCategoriesModel
 
 from .api_worker import process_custom_request
 from .auth import AuthUser, UserRole, staff_required
@@ -816,3 +824,52 @@ async def process_custom_requests(
         kwargs={"user": user.model_dump()},
     )
     return JSONResponse({"task_id": task.id, "track_link": f"/tasks/status/{task.id}/"})
+
+
+@router.post(
+    "/snapshot/yaml/",
+    openapi_extra={
+        "requestBody": {
+            "content": {
+                "application/x-yaml": {"schema": CustomRequestsYaml.model_json_schema()}
+            },
+            "required": True,
+        },
+    },
+)
+@limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute")
+@version(1)
+async def process_custom_requests_yaml(
+    request: Request,
+    user: AuthUser = Depends(staff_required),
+):
+    raw_body = await request.body()
+    try:
+        data = yaml.safe_load(raw_body)
+    except yaml.YAMLError:
+        raise HTTPException(status_code=422, detail="Invalid YAML")
+    try:
+        validated_data = DynamicCategoriesModel.model_validate(data)
+    except ValidationError as e:
+        raise HTTPException(status_code=422, detail=e.errors(include_url=False))
+
+    queue_name = validated_data.queue
+    if validated_data.queue != DEFAULT_QUEUE_NAME and user.role != UserRole.ADMIN.value:
+        raise HTTPException(
+            status_code=403,
+            detail=[{"msg": "Insufficient Permission to choose queue"}],
+        )
+    validated_data.categories = [
+        category for category in validated_data.categories if category
+    ]
+    if len(validated_data.categories) == 0:
+        raise HTTPException(
+            status_code=400, detail=[{"msg": "Categories can't be empty"}]
+        )
+    task = process_custom_request.apply_async(
+        args=(validated_data.model_dump(),),
+        queue=queue_name,
+        track_started=True,
+        kwargs={"user": user.model_dump()},
+    )
+    return JSONResponse({"task_id": task.id, "track_link": f"/tasks/status/{task.id}/"})

requirements.txt

@@ -26,7 +26,7 @@ osm-login-python==1.0.2
 humanize==4.9.0
 python-slugify==8.0.1
 geomet==1.1.0
-
+PyYAML==6.0.1
 ## documentation
 # mkdocs-material==8.5.11
 # mkdocs-jupyter==0.22.0

src/validation/models.py

@@ -18,17 +18,13 @@
 # <[email protected]>
 """Page contains validation models for application"""
 # Standard library imports
-import json
 from enum import Enum
 from typing import Dict, List, Optional, Union
 
 # Third party imports
-from area import area
 from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Polygon
-from geojson_pydantic.types import BBox
 from pydantic import BaseModel as PydanticModel
 from pydantic import Field, validator
-from typing_extensions import TypedDict
 
 # Reader imports
 from src.config import (
@@ -560,32 +556,19 @@ def validate_frequency(cls, value):
         return value.strip()
 
 
-class DynamicCategoriesModel(BaseModel, GeometryValidatorMixin):
-    """
-    Model for dynamic categories.
-
-    Fields:
-    - iso3 (Optional[str]): ISO3 Country Code.
-    - dataset (Optional[DatasetConfig]): Dataset Configurations for HDX Upload.
-    - meta (bool): Dumps Meta db in parquet format & HDX config JSON to S3.
-    - hdx_upload (bool): Enable/Disable uploading the dataset to HDX.
-    - categories (List[Dict[str, CategoryModel]]): List of dynamic categories.
-    - geometry (Optional[Union[Polygon, MultiPolygon]]): Custom polygon geometry.
-    """
-
-    iso3: Optional[str] = Field(
-        default=None,
-        description="ISO3 Country Code",
-        min_length=3,
-        max_length=3,
-        example="USA",
-    )
+class CategoriesBase(BaseModel):
     hdx_upload: bool = Field(
         default=False,
         description="Enable/Disable uploading dataset to hdx, False by default",
     )
     dataset: Optional[DatasetConfig] = Field(
-        default=None, description="Dataset Configurations for HDX Upload"
+        default=None,
+        description="Dataset Configurations for HDX Upload",
+        example={
+            "dataset_prefix": "hotosm_project_1",
+            "dataset_folder": "TM",
+            "dataset_title": "Tasking Manger Project 1",
+        },
     )
     queue: Optional[str] = Field(
         default="raw_ondemand",
@@ -607,12 +590,33 @@ class DynamicCategoriesModel(BaseModel, GeometryValidatorMixin):
                     },
                     "types": ["lines", "polygons"],
                     "select": ["name", "highway"],
-                    "where": "highway IS NOT NULL",
+                    "where": "tags['highway'] IS NOT NULL",
                     "formats": ["geojson"],
                 }
             }
         ],
     )
+
+class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin):
+    """
+    Model for dynamic categories.
+
+    Fields:
+    - iso3 (Optional[str]): ISO3 Country Code.
+    - dataset (Optional[DatasetConfig]): Dataset Configurations for HDX Upload.
+    - meta (bool): Dumps Meta db in parquet format & HDX config JSON to S3.
+    - hdx_upload (bool): Enable/Disable uploading the dataset to HDX.
+    - categories (List[Dict[str, CategoryModel]]): List of dynamic categories.
+    - geometry (Optional[Union[Polygon, MultiPolygon]]): Custom polygon geometry.
+    """
+
+    iso3: Optional[str] = Field(
+        default=None,
+        description="ISO3 Country Code",
+        min_length=3,
+        max_length=3,
+        example="USA",
+    )
     geometry: Optional[
         Union[Polygon, MultiPolygon, Feature, FeatureCollection]
     ] = Field(
@@ -653,3 +657,21 @@ def set_geometry_or_iso3(cls, value, values):
                     if item is None:
                         raise ValueError(f"Missing, Dataset config : {item}")
         return value
+
+
+class CustomRequestsYaml(CategoriesBase):
+    geometry: Union[Polygon, MultiPolygon, Feature, FeatureCollection] = Field(
+        default=None,
+        example={
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [83.96919250488281, 28.194446860487773],
+                    [83.99751663208006, 28.194446860487773],
+                    [83.99751663208006, 28.214869548073377],
+                    [83.96919250488281, 28.214869548073377],
+                    [83.96919250488281, 28.194446860487773],
+                ]
+            ],
+        },
+    )