feat(api): Add S3 backup functionality to backend

Co-authored-by: elisiariocouto <818914+elisiariocouto@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2025-09-23 00:23:56 +00:00
committed by Elisiário Couto
parent 704c3d4cb7
commit 7f2a4634c5
9 changed files with 598 additions and 1 deletions

View File

@@ -28,3 +28,13 @@ enabled = true
[filters]
case_insensitive = ["salary", "utility"]
case_sensitive = ["SpecificStore"]
# Optional: S3 backup configuration
[backup.s3]
access_key_id = "your-s3-access-key"
secret_access_key = "your-s3-secret-key"
bucket_name = "your-bucket-name"
region = "us-east-1"
# endpoint_url = "https://custom-s3-endpoint.com" # Optional: for custom S3-compatible endpoints
path_style = false # Set to true for path-style addressing
enabled = true

View File

@@ -0,0 +1,45 @@
"""API models for backup endpoints."""
from typing import Optional
from pydantic import BaseModel, Field
class S3Config(BaseModel):
"""S3 backup configuration model for API."""
access_key_id: str = Field(..., description="AWS S3 access key ID")
secret_access_key: str = Field(..., description="AWS S3 secret access key")
bucket_name: str = Field(..., description="S3 bucket name")
region: str = Field(default="us-east-1", description="AWS S3 region")
endpoint_url: Optional[str] = Field(default=None, description="Custom S3 endpoint URL")
path_style: bool = Field(default=False, description="Use path-style addressing")
enabled: bool = Field(default=True, description="Enable S3 backups")
class BackupSettings(BaseModel):
"""Backup settings model for API."""
s3: Optional[S3Config] = None
class BackupTest(BaseModel):
"""Backup connection test request model."""
service: str = Field(..., description="Backup service type (s3)")
config: S3Config = Field(..., description="S3 configuration to test")
class BackupInfo(BaseModel):
"""Backup file information model."""
key: str = Field(..., description="S3 object key")
last_modified: str = Field(..., description="Last modified timestamp (ISO format)")
size: int = Field(..., description="File size in bytes")
class BackupOperation(BaseModel):
"""Backup operation request model."""
operation: str = Field(..., description="Operation type (backup, restore)")
backup_key: Optional[str] = Field(default=None, description="Backup key for restore operations")

258
leggen/api/routes/backup.py Normal file
View File

@@ -0,0 +1,258 @@
"""API routes for backup management."""
from fastapi import APIRouter, HTTPException
from loguru import logger
from leggen.api.models.backup import (
BackupOperation,
BackupSettings,
BackupTest,
S3Config,
)
from leggen.api.models.common import APIResponse
from leggen.models.config import S3BackupConfig
from leggen.services.backup_service import BackupService
from leggen.utils.config import config
from leggen.utils.paths import path_manager
router = APIRouter()
@router.get("/backup/settings", response_model=APIResponse)
async def get_backup_settings() -> APIResponse:
"""Get current backup settings."""
try:
backup_config = config.backup_config
# Build response safely without exposing secrets
s3_config = backup_config.get("s3", {})
settings = BackupSettings(
s3=S3Config(
access_key_id="***" if s3_config.get("access_key_id") else "",
secret_access_key="***" if s3_config.get("secret_access_key") else "",
bucket_name=s3_config.get("bucket_name", ""),
region=s3_config.get("region", "us-east-1"),
endpoint_url=s3_config.get("endpoint_url"),
path_style=s3_config.get("path_style", False),
enabled=s3_config.get("enabled", True),
)
if s3_config.get("bucket_name")
else None,
)
return APIResponse(
success=True,
data=settings,
message="Backup settings retrieved successfully",
)
except Exception as e:
logger.error(f"Failed to get backup settings: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to get backup settings: {str(e)}"
) from e
@router.put("/backup/settings", response_model=APIResponse)
async def update_backup_settings(settings: BackupSettings) -> APIResponse:
"""Update backup settings."""
try:
# First test the connection if S3 config is provided
if settings.s3:
# Convert API model to config model
s3_config = S3BackupConfig(
access_key_id=settings.s3.access_key_id,
secret_access_key=settings.s3.secret_access_key,
bucket_name=settings.s3.bucket_name,
region=settings.s3.region,
endpoint_url=settings.s3.endpoint_url,
path_style=settings.s3.path_style,
enabled=settings.s3.enabled,
)
# Test connection
backup_service = BackupService()
connection_success = await backup_service.test_connection(s3_config)
if not connection_success:
raise HTTPException(
status_code=400,
detail="S3 connection test failed. Please check your configuration."
)
# Update backup config
backup_config = {}
if settings.s3:
backup_config["s3"] = {
"access_key_id": settings.s3.access_key_id,
"secret_access_key": settings.s3.secret_access_key,
"bucket_name": settings.s3.bucket_name,
"region": settings.s3.region,
"endpoint_url": settings.s3.endpoint_url,
"path_style": settings.s3.path_style,
"enabled": settings.s3.enabled,
}
# Save to config
if backup_config:
config.update_section("backup", backup_config)
return APIResponse(
success=True,
data={"updated": True},
message="Backup settings updated successfully",
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to update backup settings: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to update backup settings: {str(e)}"
) from e
@router.post("/backup/test", response_model=APIResponse)
async def test_backup_connection(test_request: BackupTest) -> APIResponse:
"""Test backup connection."""
try:
if test_request.service != "s3":
raise HTTPException(
status_code=400, detail="Only 's3' service is supported"
)
# Convert API model to config model
s3_config = S3BackupConfig(
access_key_id=test_request.config.access_key_id,
secret_access_key=test_request.config.secret_access_key,
bucket_name=test_request.config.bucket_name,
region=test_request.config.region,
endpoint_url=test_request.config.endpoint_url,
path_style=test_request.config.path_style,
enabled=test_request.config.enabled,
)
backup_service = BackupService()
success = await backup_service.test_connection(s3_config)
if success:
return APIResponse(
success=True,
data={"connected": True},
message="S3 connection test successful",
)
else:
return APIResponse(
success=False,
message="S3 connection test failed",
)
except Exception as e:
logger.error(f"Failed to test backup connection: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to test backup connection: {str(e)}"
) from e
@router.get("/backup/list", response_model=APIResponse)
async def list_backups() -> APIResponse:
"""List available backups."""
try:
backup_config = config.backup_config.get("s3", {})
if not backup_config.get("bucket_name"):
return APIResponse(
success=True,
data=[],
message="No S3 backup configuration found",
)
# Convert config to model
s3_config = S3BackupConfig(**backup_config)
backup_service = BackupService(s3_config)
backups = await backup_service.list_backups()
return APIResponse(
success=True,
data=backups,
message=f"Found {len(backups)} backups",
)
except Exception as e:
logger.error(f"Failed to list backups: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to list backups: {str(e)}"
) from e
@router.post("/backup/operation", response_model=APIResponse)
async def backup_operation(operation_request: BackupOperation) -> APIResponse:
"""Perform backup operation (backup or restore)."""
try:
backup_config = config.backup_config.get("s3", {})
if not backup_config.get("bucket_name"):
raise HTTPException(
status_code=400, detail="S3 backup is not configured"
)
# Convert config to model
s3_config = S3BackupConfig(**backup_config)
backup_service = BackupService(s3_config)
if operation_request.operation == "backup":
# Backup database
database_path = path_manager.get_database_path()
success = await backup_service.backup_database(database_path)
if success:
return APIResponse(
success=True,
data={"operation": "backup", "completed": True},
message="Database backup completed successfully",
)
else:
return APIResponse(
success=False,
message="Database backup failed",
)
elif operation_request.operation == "restore":
if not operation_request.backup_key:
raise HTTPException(
status_code=400, detail="backup_key is required for restore operation"
)
# Restore database
database_path = path_manager.get_database_path()
success = await backup_service.restore_database(
operation_request.backup_key, database_path
)
if success:
return APIResponse(
success=True,
data={"operation": "restore", "completed": True},
message="Database restore completed successfully",
)
else:
return APIResponse(
success=False,
message="Database restore failed",
)
else:
raise HTTPException(
status_code=400, detail="Invalid operation. Use 'backup' or 'restore'"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to perform backup operation: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to perform backup operation: {str(e)}"
) from e

View File

@@ -7,7 +7,7 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from loguru import logger
from leggen.api.routes import accounts, banks, notifications, sync, transactions
from leggen.api.routes import accounts, banks, backup, notifications, sync, transactions
from leggen.background.scheduler import scheduler
from leggen.utils.config import config
from leggen.utils.paths import path_manager
@@ -81,6 +81,7 @@ def create_app() -> FastAPI:
app.include_router(transactions.router, prefix="/api/v1", tags=["transactions"])
app.include_router(sync.router, prefix="/api/v1", tags=["sync"])
app.include_router(notifications.router, prefix="/api/v1", tags=["notifications"])
app.include_router(backup.router, prefix="/api/v1", tags=["backup"])
@app.get("/api/v1/health")
async def health():

View File

@@ -32,6 +32,20 @@ class NotificationConfig(BaseModel):
telegram: Optional[TelegramNotificationConfig] = None
class S3BackupConfig(BaseModel):
access_key_id: str = Field(..., description="AWS S3 access key ID")
secret_access_key: str = Field(..., description="AWS S3 secret access key")
bucket_name: str = Field(..., description="S3 bucket name")
region: str = Field(default="us-east-1", description="AWS S3 region")
endpoint_url: Optional[str] = Field(default=None, description="Custom S3 endpoint URL")
path_style: bool = Field(default=False, description="Use path-style addressing")
enabled: bool = Field(default=True, description="Enable S3 backups")
class BackupConfig(BaseModel):
s3: Optional[S3BackupConfig] = None
class FilterConfig(BaseModel):
case_insensitive: Optional[List[str]] = Field(default_factory=list)
case_sensitive: Optional[List[str]] = Field(default_factory=list)
@@ -56,3 +70,4 @@ class Config(BaseModel):
notifications: Optional[NotificationConfig] = None
filters: Optional[FilterConfig] = None
scheduler: SchedulerConfig = Field(default_factory=SchedulerConfig)
backup: Optional[BackupConfig] = None

View File

@@ -0,0 +1,190 @@
"""Backup service for S3 storage."""
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional
import boto3
from botocore.exceptions import ClientError, NoCredentialsError
from loguru import logger
from leggen.models.config import S3BackupConfig
class BackupService:
"""Service for managing S3 backups."""
def __init__(self, s3_config: Optional[S3BackupConfig] = None):
"""Initialize backup service with S3 configuration."""
self.s3_config = s3_config
self._s3_client = None
def _get_s3_client(self, config: Optional[S3BackupConfig] = None):
"""Get or create S3 client with current configuration."""
current_config = config or self.s3_config
if not current_config:
raise ValueError("S3 configuration is required")
# Create S3 client with configuration
session = boto3.Session(
aws_access_key_id=current_config.access_key_id,
aws_secret_access_key=current_config.secret_access_key,
region_name=current_config.region,
)
s3_kwargs = {}
if current_config.endpoint_url:
s3_kwargs["endpoint_url"] = current_config.endpoint_url
if current_config.path_style:
s3_kwargs["config"] = boto3.client("s3").meta.config
s3_kwargs["config"].s3 = {"addressing_style": "path"}
return session.client("s3", **s3_kwargs)
async def test_connection(self, config: S3BackupConfig) -> bool:
"""Test S3 connection with provided configuration.
Args:
config: S3 configuration to test
Returns:
True if connection successful, False otherwise
"""
try:
s3_client = self._get_s3_client(config)
# Try to list objects in the bucket (limited to 1 to minimize cost)
s3_client.list_objects_v2(Bucket=config.bucket_name, MaxKeys=1)
logger.info(f"S3 connection test successful for bucket: {config.bucket_name}")
return True
except NoCredentialsError:
logger.error("S3 credentials not found or invalid")
return False
except ClientError as e:
error_code = e.response["Error"]["Code"]
logger.error(f"S3 connection test failed: {error_code} - {e.response['Error']['Message']}")
return False
except Exception as e:
logger.error(f"Unexpected error during S3 connection test: {str(e)}")
return False
async def backup_database(self, database_path: Path) -> bool:
"""Backup database file to S3.
Args:
database_path: Path to the SQLite database file
Returns:
True if backup successful, False otherwise
"""
if not self.s3_config or not self.s3_config.enabled:
logger.warning("S3 backup is not configured or disabled")
return False
if not database_path.exists():
logger.error(f"Database file not found: {database_path}")
return False
try:
s3_client = self._get_s3_client()
# Generate backup filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_key = f"leggen_backups/database_backup_{timestamp}.db"
# Upload database file
logger.info(f"Starting database backup to S3: {backup_key}")
s3_client.upload_file(
str(database_path),
self.s3_config.bucket_name,
backup_key
)
logger.info(f"Database backup completed successfully: {backup_key}")
return True
except Exception as e:
logger.error(f"Database backup failed: {str(e)}")
return False
async def list_backups(self) -> list[dict]:
"""List available backups in S3.
Returns:
List of backup metadata dictionaries
"""
if not self.s3_config or not self.s3_config.enabled:
logger.warning("S3 backup is not configured or disabled")
return []
try:
s3_client = self._get_s3_client()
# List objects with backup prefix
response = s3_client.list_objects_v2(
Bucket=self.s3_config.bucket_name,
Prefix="leggen_backups/"
)
backups = []
for obj in response.get("Contents", []):
backups.append({
"key": obj["Key"],
"last_modified": obj["LastModified"].isoformat(),
"size": obj["Size"],
})
# Sort by last modified (newest first)
backups.sort(key=lambda x: x["last_modified"], reverse=True)
return backups
except Exception as e:
logger.error(f"Failed to list backups: {str(e)}")
return []
async def restore_database(self, backup_key: str, restore_path: Path) -> bool:
"""Restore database from S3 backup.
Args:
backup_key: S3 key of the backup to restore
restore_path: Path where to restore the database
Returns:
True if restore successful, False otherwise
"""
if not self.s3_config or not self.s3_config.enabled:
logger.warning("S3 backup is not configured or disabled")
return False
try:
s3_client = self._get_s3_client()
# Download backup file
logger.info(f"Starting database restore from S3: {backup_key}")
# Create parent directory if it doesn't exist
restore_path.parent.mkdir(parents=True, exist_ok=True)
# Download to temporary file first, then move to final location
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
s3_client.download_file(
self.s3_config.bucket_name,
backup_key,
temp_file.name
)
# Move temp file to final location
temp_path = Path(temp_file.name)
temp_path.replace(restore_path)
logger.info(f"Database restore completed successfully: {restore_path}")
return True
except Exception as e:
logger.error(f"Database restore failed: {str(e)}")
return False

View File

@@ -162,6 +162,11 @@ class Config:
}
return self.config.get("scheduler", default_schedule)
@property
def backup_config(self) -> Dict[str, Any]:
"""Get backup configuration"""
return self.config.get("backup", {})
def load_config(ctx: click.Context, _, filename):
try:

View File

@@ -35,6 +35,7 @@ dependencies = [
"tomli-w>=1.0.0,<2",
"httpx>=0.28.1",
"pydantic>=2.0.0,<3",
"boto3>=1.35.0,<2",
]
[project.urls]

72
uv.lock generated
View File

@@ -36,6 +36,34 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d0/ae/9a053dd9229c0fde6b1f1f33f609ccff1ee79ddda364c756a924c6d8563b/APScheduler-3.11.0-py3-none-any.whl", hash = "sha256:fc134ca32e50f5eadcc4938e3a4545ab19131435e851abb40b34d63d5141c6da", size = 64004, upload-time = "2024-11-24T19:39:24.442Z" },
]
[[package]]
name = "boto3"
version = "1.40.36"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore" },
{ name = "jmespath" },
{ name = "s3transfer" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8d/21/7bc857b155e8264c92b6fa8e0860a67dc01a19cbe6ba4342500299f2ae5b/boto3-1.40.36.tar.gz", hash = "sha256:bfc1f3d5c4f5d12b8458406b8972f8794ac57e2da1ee441469e143bc0440a5c3", size = 111552, upload-time = "2025-09-22T19:26:17.357Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/70/4c/428b728d5cf9003f83f735d10dd522945ab20c7d67e6c987909f29be12a0/boto3-1.40.36-py3-none-any.whl", hash = "sha256:d7c1fe033f491f560cd26022a9dcf28baf877ae854f33bc64fffd0df3b9c98be", size = 139345, upload-time = "2025-09-22T19:26:15.194Z" },
]
[[package]]
name = "botocore"
version = "1.40.36"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "jmespath" },
{ name = "python-dateutil" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4b/30/75fdc75933d3bc1c8dd7fbaee771438328b518936906b411075b1eacac93/botocore-1.40.36.tar.gz", hash = "sha256:93386a8dc54173267ddfc6cd8636c9171e021f7c032aa1df3af7de816e3df616", size = 14349583, upload-time = "2025-09-22T19:26:05.957Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d8/51/95c0324ac20b5bbafad4c89dd610c8e0dd6cbadbb2c8ca66dc95ccde98b8/botocore-1.40.36-py3-none-any.whl", hash = "sha256:d6edf75875e4013cb7078875a1d6c289afb4cc6675d99d80700c692d8d8e0b72", size = 14020478, upload-time = "2025-09-22T19:26:02.054Z" },
]
[[package]]
name = "certifi"
version = "2025.8.3"
@@ -218,12 +246,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
]
[[package]]
name = "jmespath"
version = "1.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
]
[[package]]
name = "leggen"
version = "2025.9.24"
source = { editable = "." }
dependencies = [
{ name = "apscheduler" },
{ name = "boto3" },
{ name = "click" },
{ name = "discord-webhook" },
{ name = "fastapi" },
@@ -253,6 +291,7 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "apscheduler", specifier = ">=3.10.0,<4" },
{ name = "boto3", specifier = ">=1.35.0,<2" },
{ name = "click", specifier = ">=8.1.7,<9" },
{ name = "discord-webhook", specifier = ">=1.3.1,<2" },
{ name = "fastapi", specifier = ">=0.104.0,<1" },
@@ -474,6 +513,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2b/b3/7fefc43fb706380144bcd293cc6e446e6f637ddfa8b83f48d1734156b529/pytest_mock-3.15.0-py3-none-any.whl", hash = "sha256:ef2219485fb1bd256b00e7ad7466ce26729b30eadfc7cbcdb4fa9a92ca68db6f", size = 10050, upload-time = "2025-09-04T20:57:47.274Z" },
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]]
name = "python-dotenv"
version = "1.1.1"
@@ -565,6 +616,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e1/a3/03216a6a86c706df54422612981fb0f9041dbb452c3401501d4a22b942c9/ruff-0.13.0-py3-none-win_arm64.whl", hash = "sha256:ab80525317b1e1d38614addec8ac954f1b3e662de9d59114ecbf771d00cf613e", size = 12312357, upload-time = "2025-09-10T16:25:35.595Z" },
]
[[package]]
name = "s3transfer"
version = "0.14.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore" },
]
sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" },
]
[[package]]
name = "six"
version = "1.17.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
]
[[package]]
name = "sniffio"
version = "1.3.1"