chore: initial snapshot for gitea/github upload
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
"""Destination implementations for Focus export."""
|
||||
|
||||
from .base import FocusDestination, FocusTimeWindow
|
||||
from .factory import FocusDestinationFactory
|
||||
from .s3_destination import FocusS3Destination
|
||||
|
||||
__all__ = [
|
||||
"FocusDestination",
|
||||
"FocusDestinationFactory",
|
||||
"FocusTimeWindow",
|
||||
"FocusS3Destination",
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Abstract destination interfaces for Focus export."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FocusTimeWindow:
|
||||
"""Represents the span of data exported in a single batch."""
|
||||
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
frequency: str
|
||||
|
||||
|
||||
class FocusDestination(Protocol):
|
||||
"""Protocol for anything that can receive Focus export files."""
|
||||
|
||||
async def deliver(
|
||||
self,
|
||||
*,
|
||||
content: bytes,
|
||||
time_window: FocusTimeWindow,
|
||||
filename: str,
|
||||
) -> None:
|
||||
"""Persist the serialized export for the provided time window."""
|
||||
...
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Factory helpers for Focus export destinations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .base import FocusDestination
|
||||
from .s3_destination import FocusS3Destination
|
||||
|
||||
|
||||
class FocusDestinationFactory:
|
||||
"""Builds destination instances based on provider/config settings."""
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
*,
|
||||
provider: str,
|
||||
prefix: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> FocusDestination:
|
||||
"""Return a destination implementation for the requested provider."""
|
||||
provider_lower = provider.lower()
|
||||
normalized_config = FocusDestinationFactory._resolve_config(
|
||||
provider=provider_lower, overrides=config or {}
|
||||
)
|
||||
if provider_lower == "s3":
|
||||
return FocusS3Destination(prefix=prefix, config=normalized_config)
|
||||
raise NotImplementedError(
|
||||
f"Provider '{provider}' not supported for Focus export"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_config(
|
||||
*,
|
||||
provider: str,
|
||||
overrides: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
if provider == "s3":
|
||||
resolved = {
|
||||
"bucket_name": overrides.get("bucket_name")
|
||||
or os.getenv("FOCUS_S3_BUCKET_NAME"),
|
||||
"region_name": overrides.get("region_name")
|
||||
or os.getenv("FOCUS_S3_REGION_NAME"),
|
||||
"endpoint_url": overrides.get("endpoint_url")
|
||||
or os.getenv("FOCUS_S3_ENDPOINT_URL"),
|
||||
"aws_access_key_id": overrides.get("aws_access_key_id")
|
||||
or os.getenv("FOCUS_S3_ACCESS_KEY"),
|
||||
"aws_secret_access_key": overrides.get("aws_secret_access_key")
|
||||
or os.getenv("FOCUS_S3_SECRET_KEY"),
|
||||
"aws_session_token": overrides.get("aws_session_token")
|
||||
or os.getenv("FOCUS_S3_SESSION_TOKEN"),
|
||||
}
|
||||
if not resolved.get("bucket_name"):
|
||||
raise ValueError("FOCUS_S3_BUCKET_NAME must be provided for S3 exports")
|
||||
return {k: v for k, v in resolved.items() if v is not None}
|
||||
raise NotImplementedError(
|
||||
f"Provider '{provider}' not supported for Focus export configuration"
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""S3 destination implementation for Focus export."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
import boto3
|
||||
|
||||
from .base import FocusDestination, FocusTimeWindow
|
||||
|
||||
|
||||
class FocusS3Destination(FocusDestination):
|
||||
"""Handles uploading serialized exports to S3 buckets."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
prefix: str,
|
||||
config: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
config = config or {}
|
||||
bucket_name = config.get("bucket_name")
|
||||
if not bucket_name:
|
||||
raise ValueError("bucket_name must be provided for S3 destination")
|
||||
self.bucket_name = bucket_name
|
||||
self.prefix = prefix.rstrip("/")
|
||||
self.config = config
|
||||
|
||||
async def deliver(
|
||||
self,
|
||||
*,
|
||||
content: bytes,
|
||||
time_window: FocusTimeWindow,
|
||||
filename: str,
|
||||
) -> None:
|
||||
object_key = self._build_object_key(time_window=time_window, filename=filename)
|
||||
await asyncio.to_thread(self._upload, content, object_key)
|
||||
|
||||
def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str:
|
||||
start_utc = time_window.start_time.astimezone(timezone.utc)
|
||||
date_component = f"date={start_utc.strftime('%Y-%m-%d')}"
|
||||
parts = [self.prefix, date_component]
|
||||
if time_window.frequency == "hourly":
|
||||
parts.append(f"hour={start_utc.strftime('%H')}")
|
||||
key_prefix = "/".join(filter(None, parts))
|
||||
return f"{key_prefix}/{filename}" if key_prefix else filename
|
||||
|
||||
def _upload(self, content: bytes, object_key: str) -> None:
|
||||
client_kwargs: dict[str, Any] = {}
|
||||
region_name = self.config.get("region_name")
|
||||
if region_name:
|
||||
client_kwargs["region_name"] = region_name
|
||||
endpoint_url = self.config.get("endpoint_url")
|
||||
if endpoint_url:
|
||||
client_kwargs["endpoint_url"] = endpoint_url
|
||||
|
||||
session_kwargs: dict[str, Any] = {}
|
||||
for key in (
|
||||
"aws_access_key_id",
|
||||
"aws_secret_access_key",
|
||||
"aws_session_token",
|
||||
):
|
||||
if self.config.get(key):
|
||||
session_kwargs[key] = self.config[key]
|
||||
|
||||
s3_client = boto3.client("s3", **client_kwargs, **session_kwargs)
|
||||
s3_client.put_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=object_key,
|
||||
Body=content,
|
||||
ContentType="application/octet-stream",
|
||||
)
|
||||
Reference in New Issue
Block a user