mirror of
https://github.com/elisiariocouto/leggen.git
synced 2025-12-25 03:49:20 +00:00
refactor: Unify leggen and leggend packages into single leggen package
- Merge leggend API components into leggen (api/, services/, background/) - Replace leggend command with 'leggen server' subcommand - Consolidate configuration systems into leggen.utils.config - Update environment variables: LEGGEND_API_URL -> LEGGEN_API_URL - Rename LeggendAPIClient -> LeggenAPIClient - Update all documentation, Docker configs, and compose files - Fix all import statements and test references - Remove duplicate utility files and clean up package structure All tests passing (101/101), linting clean, server functionality preserved. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Elisiário Couto
committed by
Elisiário Couto
Elisiário Couto
parent
0e645d9bae
commit
318ca517f7
903
leggen/services/database_service.py
Normal file
903
leggen/services/database_service.py
Normal file
@@ -0,0 +1,903 @@
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
import sqlite3
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from leggen.utils.config import config
|
||||
import leggen.database.sqlite as sqlite_db
|
||||
from leggen.utils.paths import path_manager
|
||||
|
||||
|
||||
class DatabaseService:
|
||||
def __init__(self):
|
||||
self.db_config = config.database_config
|
||||
self.sqlite_enabled = self.db_config.get("sqlite", True)
|
||||
|
||||
async def persist_balance(
|
||||
self, account_id: str, balance_data: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Persist account balance data"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, skipping balance persistence")
|
||||
return
|
||||
|
||||
await self._persist_balance_sqlite(account_id, balance_data)
|
||||
|
||||
async def persist_transactions(
|
||||
self, account_id: str, transactions: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Persist transactions and return new transactions"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, skipping transaction persistence")
|
||||
return transactions
|
||||
|
||||
return await self._persist_transactions_sqlite(account_id, transactions)
|
||||
|
||||
def process_transactions(
|
||||
self,
|
||||
account_id: str,
|
||||
account_info: Dict[str, Any],
|
||||
transaction_data: Dict[str, Any],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Process raw transaction data into standardized format"""
|
||||
transactions = []
|
||||
|
||||
# Process booked transactions
|
||||
for transaction in transaction_data.get("transactions", {}).get("booked", []):
|
||||
processed = self._process_single_transaction(
|
||||
account_id, account_info, transaction, "booked"
|
||||
)
|
||||
transactions.append(processed)
|
||||
|
||||
# Process pending transactions
|
||||
for transaction in transaction_data.get("transactions", {}).get("pending", []):
|
||||
processed = self._process_single_transaction(
|
||||
account_id, account_info, transaction, "pending"
|
||||
)
|
||||
transactions.append(processed)
|
||||
|
||||
return transactions
|
||||
|
||||
def _process_single_transaction(
|
||||
self,
|
||||
account_id: str,
|
||||
account_info: Dict[str, Any],
|
||||
transaction: Dict[str, Any],
|
||||
status: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Process a single transaction into standardized format"""
|
||||
# Extract dates
|
||||
booked_date = transaction.get("bookingDateTime") or transaction.get(
|
||||
"bookingDate"
|
||||
)
|
||||
value_date = transaction.get("valueDateTime") or transaction.get("valueDate")
|
||||
|
||||
if booked_date and value_date:
|
||||
min_date = min(
|
||||
datetime.fromisoformat(booked_date), datetime.fromisoformat(value_date)
|
||||
)
|
||||
else:
|
||||
date_str = booked_date or value_date
|
||||
if not date_str:
|
||||
raise ValueError("No valid date found in transaction")
|
||||
min_date = datetime.fromisoformat(date_str)
|
||||
|
||||
# Extract amount and currency
|
||||
transaction_amount = transaction.get("transactionAmount", {})
|
||||
amount = float(transaction_amount.get("amount", 0))
|
||||
currency = transaction_amount.get("currency", "")
|
||||
|
||||
# Extract description
|
||||
description = transaction.get(
|
||||
"remittanceInformationUnstructured",
|
||||
",".join(transaction.get("remittanceInformationUnstructuredArray", [])),
|
||||
)
|
||||
|
||||
# Extract transaction IDs - transactionId is now primary, internalTransactionId is reference
|
||||
transaction_id = transaction.get("transactionId")
|
||||
internal_transaction_id = transaction.get("internalTransactionId")
|
||||
|
||||
if not transaction_id:
|
||||
raise ValueError("Transaction missing required transactionId field")
|
||||
|
||||
return {
|
||||
"accountId": account_id,
|
||||
"transactionId": transaction_id,
|
||||
"internalTransactionId": internal_transaction_id,
|
||||
"institutionId": account_info["institution_id"],
|
||||
"iban": account_info.get("iban", "N/A"),
|
||||
"transactionDate": min_date,
|
||||
"description": description,
|
||||
"transactionValue": amount,
|
||||
"transactionCurrency": currency,
|
||||
"transactionStatus": status,
|
||||
"rawTransaction": transaction,
|
||||
}
|
||||
|
||||
async def get_transactions_from_db(
|
||||
self,
|
||||
account_id: Optional[str] = None,
|
||||
limit: Optional[int] = None, # None means no limit, used for stats
|
||||
offset: Optional[int] = 0,
|
||||
date_from: Optional[str] = None,
|
||||
date_to: Optional[str] = None,
|
||||
min_amount: Optional[float] = None,
|
||||
max_amount: Optional[float] = None,
|
||||
search: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get transactions from SQLite database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, cannot read transactions")
|
||||
return []
|
||||
|
||||
try:
|
||||
transactions = sqlite_db.get_transactions(
|
||||
account_id=account_id,
|
||||
limit=limit, # Pass limit as-is, None means no limit
|
||||
offset=offset or 0,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
min_amount=min_amount,
|
||||
max_amount=max_amount,
|
||||
search=search,
|
||||
)
|
||||
logger.debug(f"Retrieved {len(transactions)} transactions from database")
|
||||
return transactions
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get transactions from database: {e}")
|
||||
return []
|
||||
|
||||
async def get_transaction_count_from_db(
|
||||
self,
|
||||
account_id: Optional[str] = None,
|
||||
date_from: Optional[str] = None,
|
||||
date_to: Optional[str] = None,
|
||||
min_amount: Optional[float] = None,
|
||||
max_amount: Optional[float] = None,
|
||||
search: Optional[str] = None,
|
||||
) -> int:
|
||||
"""Get total count of transactions from SQLite database"""
|
||||
if not self.sqlite_enabled:
|
||||
return 0
|
||||
|
||||
try:
|
||||
filters = {
|
||||
"date_from": date_from,
|
||||
"date_to": date_to,
|
||||
"min_amount": min_amount,
|
||||
"max_amount": max_amount,
|
||||
"search": search,
|
||||
}
|
||||
# Remove None values
|
||||
filters = {k: v for k, v in filters.items() if v is not None}
|
||||
|
||||
count = sqlite_db.get_transaction_count(account_id=account_id, **filters)
|
||||
logger.debug(f"Total transaction count: {count}")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get transaction count from database: {e}")
|
||||
return 0
|
||||
|
||||
async def get_balances_from_db(
|
||||
self, account_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get balances from SQLite database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, cannot read balances")
|
||||
return []
|
||||
|
||||
try:
|
||||
balances = sqlite_db.get_balances(account_id=account_id)
|
||||
logger.debug(f"Retrieved {len(balances)} balances from database")
|
||||
return balances
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get balances from database: {e}")
|
||||
return []
|
||||
|
||||
async def get_historical_balances_from_db(
|
||||
self, account_id: Optional[str] = None, days: int = 365
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get historical balance progression from SQLite database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, cannot read historical balances")
|
||||
return []
|
||||
|
||||
try:
|
||||
balances = sqlite_db.get_historical_balances(
|
||||
account_id=account_id, days=days
|
||||
)
|
||||
logger.debug(
|
||||
f"Retrieved {len(balances)} historical balance points from database"
|
||||
)
|
||||
return balances
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get historical balances from database: {e}")
|
||||
return []
|
||||
|
||||
async def get_account_summary_from_db(
|
||||
self, account_id: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Get basic account info from SQLite database (avoids GoCardless call)"""
|
||||
if not self.sqlite_enabled:
|
||||
return None
|
||||
|
||||
try:
|
||||
summary = sqlite_db.get_account_summary(account_id)
|
||||
if summary:
|
||||
logger.debug(
|
||||
f"Retrieved account summary from database for {account_id}"
|
||||
)
|
||||
return summary
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get account summary from database: {e}")
|
||||
return None
|
||||
|
||||
async def persist_account_details(self, account_data: Dict[str, Any]) -> None:
|
||||
"""Persist account details to database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, skipping account persistence")
|
||||
return
|
||||
|
||||
await self._persist_account_details_sqlite(account_data)
|
||||
|
||||
async def get_accounts_from_db(
|
||||
self, account_ids: Optional[List[str]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get account details from database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, cannot read accounts")
|
||||
return []
|
||||
|
||||
try:
|
||||
accounts = sqlite_db.get_accounts(account_ids=account_ids)
|
||||
logger.debug(f"Retrieved {len(accounts)} accounts from database")
|
||||
return accounts
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get accounts from database: {e}")
|
||||
return []
|
||||
|
||||
async def get_account_details_from_db(
|
||||
self, account_id: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific account details from database"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.warning("SQLite database disabled, cannot read account")
|
||||
return None
|
||||
|
||||
try:
|
||||
account = sqlite_db.get_account(account_id)
|
||||
if account:
|
||||
logger.debug(
|
||||
f"Retrieved account details from database for {account_id}"
|
||||
)
|
||||
return account
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get account details from database: {e}")
|
||||
return None
|
||||
|
||||
async def run_migrations_if_needed(self):
|
||||
"""Run all necessary database migrations"""
|
||||
if not self.sqlite_enabled:
|
||||
logger.info("SQLite database disabled, skipping migrations")
|
||||
return
|
||||
|
||||
await self._migrate_balance_timestamps_if_needed()
|
||||
await self._migrate_null_transaction_ids_if_needed()
|
||||
await self._migrate_to_composite_key_if_needed()
|
||||
|
||||
async def _migrate_balance_timestamps_if_needed(self):
|
||||
"""Check and migrate balance timestamps if needed"""
|
||||
try:
|
||||
if await self._check_balance_timestamp_migration_needed():
|
||||
logger.info("Balance timestamp migration needed, starting...")
|
||||
await self._migrate_balance_timestamps()
|
||||
logger.info("Balance timestamp migration completed")
|
||||
else:
|
||||
logger.info("Balance timestamps are already consistent")
|
||||
except Exception as e:
|
||||
logger.error(f"Balance timestamp migration failed: {e}")
|
||||
raise
|
||||
|
||||
async def _check_balance_timestamp_migration_needed(self) -> bool:
|
||||
"""Check if balance timestamps need migration"""
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check for mixed timestamp types
|
||||
cursor.execute("""
|
||||
SELECT typeof(timestamp) as type, COUNT(*) as count
|
||||
FROM balances
|
||||
GROUP BY typeof(timestamp)
|
||||
""")
|
||||
|
||||
types = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
# If we have both 'real' and 'text' types, migration is needed
|
||||
type_names = [row[0] for row in types]
|
||||
return "real" in type_names and "text" in type_names
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check migration status: {e}")
|
||||
return False
|
||||
|
||||
async def _migrate_balance_timestamps(self):
|
||||
"""Convert all Unix timestamps to datetime strings"""
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
logger.warning("Database file not found, skipping migration")
|
||||
return
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get all balances with REAL timestamps
|
||||
cursor.execute("""
|
||||
SELECT id, timestamp
|
||||
FROM balances
|
||||
WHERE typeof(timestamp) = 'real'
|
||||
ORDER BY id
|
||||
""")
|
||||
|
||||
unix_records = cursor.fetchall()
|
||||
total_records = len(unix_records)
|
||||
|
||||
if total_records == 0:
|
||||
logger.info("No Unix timestamps found to migrate")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Migrating {total_records} balance records from Unix to datetime format"
|
||||
)
|
||||
|
||||
# Convert and update in batches
|
||||
batch_size = 100
|
||||
migrated_count = 0
|
||||
|
||||
for i in range(0, total_records, batch_size):
|
||||
batch = unix_records[i : i + batch_size]
|
||||
|
||||
for record_id, unix_timestamp in batch:
|
||||
try:
|
||||
# Convert Unix timestamp to datetime string
|
||||
dt_string = self._unix_to_datetime_string(float(unix_timestamp))
|
||||
|
||||
# Update the record
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE balances
|
||||
SET timestamp = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(dt_string, record_id),
|
||||
)
|
||||
|
||||
migrated_count += 1
|
||||
|
||||
if migrated_count % 100 == 0:
|
||||
logger.info(
|
||||
f"Migrated {migrated_count}/{total_records} balance records"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to migrate record {record_id}: {e}")
|
||||
continue
|
||||
|
||||
# Commit batch
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
logger.info(f"Successfully migrated {migrated_count} balance records")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Balance timestamp migration failed: {e}")
|
||||
raise
|
||||
|
||||
async def _migrate_null_transaction_ids_if_needed(self):
|
||||
"""Check and migrate null transaction IDs if needed"""
|
||||
try:
|
||||
if await self._check_null_transaction_ids_migration_needed():
|
||||
logger.info("Null transaction IDs migration needed, starting...")
|
||||
await self._migrate_null_transaction_ids()
|
||||
logger.info("Null transaction IDs migration completed")
|
||||
else:
|
||||
logger.info("No null transaction IDs found to migrate")
|
||||
except Exception as e:
|
||||
logger.error(f"Null transaction IDs migration failed: {e}")
|
||||
raise
|
||||
|
||||
async def _check_null_transaction_ids_migration_needed(self) -> bool:
|
||||
"""Check if null transaction IDs need migration"""
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check for transactions with null or empty internalTransactionId
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM transactions
|
||||
WHERE (internalTransactionId IS NULL OR internalTransactionId = '')
|
||||
AND json_extract(rawTransaction, '$.transactionId') IS NOT NULL
|
||||
""")
|
||||
|
||||
count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
return count > 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check null transaction IDs migration status: {e}")
|
||||
return False
|
||||
|
||||
async def _migrate_null_transaction_ids(self):
|
||||
"""Populate null internalTransactionId fields using transactionId from raw data"""
|
||||
import uuid
|
||||
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
logger.warning("Database file not found, skipping migration")
|
||||
return
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get all transactions with null/empty internalTransactionId but valid transactionId in raw data
|
||||
cursor.execute("""
|
||||
SELECT rowid, json_extract(rawTransaction, '$.transactionId') as transactionId
|
||||
FROM transactions
|
||||
WHERE (internalTransactionId IS NULL OR internalTransactionId = '')
|
||||
AND json_extract(rawTransaction, '$.transactionId') IS NOT NULL
|
||||
ORDER BY rowid
|
||||
""")
|
||||
|
||||
null_records = cursor.fetchall()
|
||||
total_records = len(null_records)
|
||||
|
||||
if total_records == 0:
|
||||
logger.info("No null transaction IDs found to migrate")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Migrating {total_records} transaction records with null internalTransactionId"
|
||||
)
|
||||
|
||||
# Update in batches
|
||||
batch_size = 100
|
||||
migrated_count = 0
|
||||
skipped_duplicates = 0
|
||||
|
||||
for i in range(0, total_records, batch_size):
|
||||
batch = null_records[i : i + batch_size]
|
||||
|
||||
for rowid, transaction_id in batch:
|
||||
try:
|
||||
# Check if this transactionId is already used by another record
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM transactions WHERE internalTransactionId = ?",
|
||||
(str(transaction_id),),
|
||||
)
|
||||
existing_count = cursor.fetchone()[0]
|
||||
|
||||
if existing_count > 0:
|
||||
# Generate a unique ID to avoid constraint violation
|
||||
unique_id = f"{str(transaction_id)}_{uuid.uuid4().hex[:8]}"
|
||||
logger.debug(
|
||||
f"Generated unique ID for duplicate transactionId: {unique_id}"
|
||||
)
|
||||
else:
|
||||
# Use the original transactionId
|
||||
unique_id = str(transaction_id)
|
||||
|
||||
# Update the record
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE transactions
|
||||
SET internalTransactionId = ?
|
||||
WHERE rowid = ?
|
||||
""",
|
||||
(unique_id, rowid),
|
||||
)
|
||||
|
||||
migrated_count += 1
|
||||
|
||||
if migrated_count % 100 == 0:
|
||||
logger.info(
|
||||
f"Migrated {migrated_count}/{total_records} transaction records"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to migrate record {rowid}: {e}")
|
||||
continue
|
||||
|
||||
# Commit batch
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
logger.info(f"Successfully migrated {migrated_count} transaction records")
|
||||
if skipped_duplicates > 0:
|
||||
logger.info(
|
||||
f"Generated unique IDs for {skipped_duplicates} duplicate transactionIds"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Null transaction IDs migration failed: {e}")
|
||||
raise
|
||||
|
||||
async def _migrate_to_composite_key_if_needed(self):
|
||||
"""Check and migrate to composite primary key if needed"""
|
||||
try:
|
||||
if await self._check_composite_key_migration_needed():
|
||||
logger.info("Composite key migration needed, starting...")
|
||||
await self._migrate_to_composite_key()
|
||||
logger.info("Composite key migration completed")
|
||||
else:
|
||||
logger.info("Composite key migration not needed")
|
||||
except Exception as e:
|
||||
logger.error(f"Composite key migration failed: {e}")
|
||||
raise
|
||||
|
||||
async def _check_composite_key_migration_needed(self) -> bool:
|
||||
"""Check if composite key migration is needed"""
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if transactions table exists
|
||||
cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='transactions'"
|
||||
)
|
||||
if not cursor.fetchone():
|
||||
conn.close()
|
||||
return False
|
||||
|
||||
# Check if transactions table has the old primary key structure
|
||||
cursor.execute("PRAGMA table_info(transactions)")
|
||||
columns = cursor.fetchall()
|
||||
|
||||
# Check if internalTransactionId is the primary key (old structure)
|
||||
internal_transaction_id_is_pk = any(
|
||||
col[1] == "internalTransactionId" and col[5] == 1 # col[5] is pk flag
|
||||
for col in columns
|
||||
)
|
||||
|
||||
# Check if we have the new composite primary key structure
|
||||
has_composite_key = any(
|
||||
col[1] in ["accountId", "transactionId"]
|
||||
and col[5] == 1 # col[5] is pk flag
|
||||
for col in columns
|
||||
)
|
||||
|
||||
conn.close()
|
||||
|
||||
# Migration is needed if:
|
||||
# 1. internalTransactionId is still the primary key (old structure), OR
|
||||
# 2. We don't have the new composite key structure yet
|
||||
return internal_transaction_id_is_pk or not has_composite_key
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check composite key migration status: {e}")
|
||||
return False
|
||||
|
||||
async def _migrate_to_composite_key(self):
|
||||
"""Migrate transactions table to use composite primary key (accountId, transactionId)"""
|
||||
db_path = path_manager.get_database_path()
|
||||
if not db_path.exists():
|
||||
logger.warning("Database file not found, skipping migration")
|
||||
return
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
logger.info("Starting composite key migration...")
|
||||
|
||||
# Step 1: Create temporary table with new schema
|
||||
logger.info("Creating temporary table with composite primary key...")
|
||||
cursor.execute("DROP TABLE IF EXISTS transactions_temp")
|
||||
cursor.execute("""
|
||||
CREATE TABLE transactions_temp (
|
||||
accountId TEXT NOT NULL,
|
||||
transactionId TEXT NOT NULL,
|
||||
internalTransactionId TEXT,
|
||||
institutionId TEXT,
|
||||
iban TEXT,
|
||||
transactionDate DATETIME,
|
||||
description TEXT,
|
||||
transactionValue REAL,
|
||||
transactionCurrency TEXT,
|
||||
transactionStatus TEXT,
|
||||
rawTransaction JSON,
|
||||
PRIMARY KEY (accountId, transactionId)
|
||||
)
|
||||
""")
|
||||
|
||||
# Step 2: Insert deduplicated data (keep most recent duplicate)
|
||||
logger.info("Inserting deduplicated data...")
|
||||
cursor.execute("""
|
||||
INSERT INTO transactions_temp
|
||||
SELECT
|
||||
accountId,
|
||||
json_extract(rawTransaction, '$.transactionId') as transactionId,
|
||||
internalTransactionId,
|
||||
institutionId,
|
||||
iban,
|
||||
transactionDate,
|
||||
description,
|
||||
transactionValue,
|
||||
transactionCurrency,
|
||||
transactionStatus,
|
||||
rawTransaction
|
||||
FROM (
|
||||
SELECT *,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY accountId, json_extract(rawTransaction, '$.transactionId')
|
||||
ORDER BY transactionDate DESC, rowid DESC
|
||||
) as rn
|
||||
FROM transactions
|
||||
WHERE json_extract(rawTransaction, '$.transactionId') IS NOT NULL
|
||||
AND accountId IS NOT NULL
|
||||
) WHERE rn = 1
|
||||
""")
|
||||
|
||||
# Get counts for reporting
|
||||
cursor.execute("SELECT COUNT(*) FROM transactions")
|
||||
old_count = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM transactions_temp")
|
||||
new_count = cursor.fetchone()[0]
|
||||
|
||||
duplicates_removed = old_count - new_count
|
||||
logger.info(
|
||||
f"Migration stats: {old_count} → {new_count} records ({duplicates_removed} duplicates removed)"
|
||||
)
|
||||
|
||||
# Step 3: Replace tables
|
||||
logger.info("Replacing tables...")
|
||||
cursor.execute("ALTER TABLE transactions RENAME TO transactions_old")
|
||||
cursor.execute("ALTER TABLE transactions_temp RENAME TO transactions")
|
||||
|
||||
# Step 4: Recreate indexes
|
||||
logger.info("Recreating indexes...")
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_transactions_internal_id ON transactions(internalTransactionId)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_transactions_date ON transactions(transactionDate)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_transactions_account_date ON transactions(accountId, transactionDate)"
|
||||
)
|
||||
cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_transactions_amount ON transactions(transactionValue)"
|
||||
)
|
||||
|
||||
# Step 5: Cleanup
|
||||
logger.info("Cleaning up...")
|
||||
cursor.execute("DROP TABLE transactions_old")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info("Composite key migration completed successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Composite key migration failed: {e}")
|
||||
raise
|
||||
|
||||
def _unix_to_datetime_string(self, unix_timestamp: float) -> str:
|
||||
"""Convert Unix timestamp to datetime string"""
|
||||
dt = datetime.fromtimestamp(unix_timestamp)
|
||||
return dt.isoformat()
|
||||
|
||||
async def _persist_balance_sqlite(
|
||||
self, account_id: str, balance_data: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Persist balance to SQLite"""
|
||||
try:
|
||||
import sqlite3
|
||||
|
||||
db_path = path_manager.get_database_path()
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Create the balances table if it doesn't exist
|
||||
cursor.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS balances (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id TEXT,
|
||||
bank TEXT,
|
||||
status TEXT,
|
||||
iban TEXT,
|
||||
amount REAL,
|
||||
currency TEXT,
|
||||
type TEXT,
|
||||
timestamp DATETIME
|
||||
)"""
|
||||
)
|
||||
|
||||
# Create indexes for better performance
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_balances_account_id
|
||||
ON balances(account_id)"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_balances_timestamp
|
||||
ON balances(timestamp)"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_balances_account_type_timestamp
|
||||
ON balances(account_id, type, timestamp)"""
|
||||
)
|
||||
|
||||
# Convert GoCardless balance format to our format and persist
|
||||
for balance in balance_data.get("balances", []):
|
||||
balance_amount = balance["balanceAmount"]
|
||||
|
||||
try:
|
||||
cursor.execute(
|
||||
"""INSERT INTO balances (
|
||||
account_id,
|
||||
bank,
|
||||
status,
|
||||
iban,
|
||||
amount,
|
||||
currency,
|
||||
type,
|
||||
timestamp
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
account_id,
|
||||
balance_data.get("institution_id", "unknown"),
|
||||
balance_data.get("account_status"),
|
||||
balance_data.get("iban", "N/A"),
|
||||
float(balance_amount["amount"]),
|
||||
balance_amount["currency"],
|
||||
balance["balanceType"],
|
||||
datetime.now().isoformat(),
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError:
|
||||
logger.warning(f"Skipped duplicate balance for {account_id}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"Persisted balances to SQLite for account {account_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to persist balances to SQLite: {e}")
|
||||
raise
|
||||
|
||||
async def _persist_transactions_sqlite(
|
||||
self, account_id: str, transactions: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Persist transactions to SQLite"""
|
||||
try:
|
||||
import sqlite3
|
||||
import json
|
||||
|
||||
db_path = path_manager.get_database_path()
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# The table should already exist with the new schema from migration
|
||||
# If it doesn't exist, create it (for new installations)
|
||||
cursor.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS transactions (
|
||||
accountId TEXT NOT NULL,
|
||||
transactionId TEXT NOT NULL,
|
||||
internalTransactionId TEXT,
|
||||
institutionId TEXT,
|
||||
iban TEXT,
|
||||
transactionDate DATETIME,
|
||||
description TEXT,
|
||||
transactionValue REAL,
|
||||
transactionCurrency TEXT,
|
||||
transactionStatus TEXT,
|
||||
rawTransaction JSON,
|
||||
PRIMARY KEY (accountId, transactionId)
|
||||
)"""
|
||||
)
|
||||
|
||||
# Create indexes for better performance (if they don't exist)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_internal_id
|
||||
ON transactions(internalTransactionId)"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_date
|
||||
ON transactions(transactionDate)"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_account_date
|
||||
ON transactions(accountId, transactionDate)"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_amount
|
||||
ON transactions(transactionValue)"""
|
||||
)
|
||||
|
||||
# Prepare an SQL statement for inserting/replacing data
|
||||
insert_sql = """INSERT OR REPLACE INTO transactions (
|
||||
accountId,
|
||||
transactionId,
|
||||
internalTransactionId,
|
||||
institutionId,
|
||||
iban,
|
||||
transactionDate,
|
||||
description,
|
||||
transactionValue,
|
||||
transactionCurrency,
|
||||
transactionStatus,
|
||||
rawTransaction
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
|
||||
|
||||
new_transactions = []
|
||||
|
||||
for transaction in transactions:
|
||||
try:
|
||||
cursor.execute(
|
||||
insert_sql,
|
||||
(
|
||||
transaction["accountId"],
|
||||
transaction["transactionId"],
|
||||
transaction.get("internalTransactionId"),
|
||||
transaction["institutionId"],
|
||||
transaction["iban"],
|
||||
transaction["transactionDate"],
|
||||
transaction["description"],
|
||||
transaction["transactionValue"],
|
||||
transaction["transactionCurrency"],
|
||||
transaction["transactionStatus"],
|
||||
json.dumps(transaction["rawTransaction"]),
|
||||
),
|
||||
)
|
||||
new_transactions.append(transaction)
|
||||
except sqlite3.IntegrityError as e:
|
||||
logger.warning(
|
||||
f"Failed to insert transaction {transaction.get('transactionId')}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(
|
||||
f"Persisted {len(new_transactions)} new transactions to SQLite for account {account_id}"
|
||||
)
|
||||
return new_transactions
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to persist transactions to SQLite: {e}")
|
||||
raise
|
||||
|
||||
async def _persist_account_details_sqlite(
|
||||
self, account_data: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Persist account details to SQLite"""
|
||||
try:
|
||||
# Use the sqlite_db module function
|
||||
sqlite_db.persist_account(account_data)
|
||||
|
||||
logger.info(
|
||||
f"Persisted account details to SQLite for account {account_data['id']}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to persist account details to SQLite: {e}")
|
||||
raise
|
||||
175
leggen/services/gocardless_service.py
Normal file
175
leggen/services/gocardless_service.py
Normal file
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import httpx
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from leggen.utils.config import config
|
||||
from leggen.utils.paths import path_manager
|
||||
|
||||
|
||||
def _log_rate_limits(response):
|
||||
"""Log GoCardless API rate limit headers"""
|
||||
limit = response.headers.get("X-RateLimit-Limit")
|
||||
remaining = response.headers.get("X-RateLimit-Remaining")
|
||||
reset = response.headers.get("X-RateLimit-Reset")
|
||||
account_success_reset = response.headers.get("X-RateLimit-Account-Success-Reset")
|
||||
|
||||
if limit or remaining or reset or account_success_reset:
|
||||
logger.info(
|
||||
f"GoCardless rate limits - Limit: {limit}, Remaining: {remaining}, Reset: {reset}s, Account Success Reset: {account_success_reset}"
|
||||
)
|
||||
|
||||
|
||||
class GoCardlessService:
|
||||
def __init__(self):
|
||||
self.config = config.gocardless_config
|
||||
self.base_url = self.config.get(
|
||||
"url", "https://bankaccountdata.gocardless.com/api/v2"
|
||||
)
|
||||
self._token = None
|
||||
|
||||
async def _get_auth_headers(self) -> Dict[str, str]:
|
||||
"""Get authentication headers for GoCardless API"""
|
||||
token = await self._get_token()
|
||||
return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
|
||||
async def _get_token(self) -> str:
|
||||
"""Get access token for GoCardless API"""
|
||||
if self._token:
|
||||
return self._token
|
||||
|
||||
# Use path manager for auth file
|
||||
auth_file = path_manager.get_auth_file_path()
|
||||
|
||||
if auth_file.exists():
|
||||
try:
|
||||
with open(auth_file, "r") as f:
|
||||
auth = json.load(f)
|
||||
|
||||
if auth.get("access"):
|
||||
# Try to refresh the token
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/token/refresh/",
|
||||
json={"refresh": auth["refresh"]},
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
auth.update(response.json())
|
||||
self._save_auth(auth)
|
||||
self._token = auth["access"]
|
||||
return self._token
|
||||
except httpx.HTTPStatusError:
|
||||
logger.warning("Token refresh failed, creating new token")
|
||||
return await self._create_token()
|
||||
else:
|
||||
return await self._create_token()
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading auth file: {e}")
|
||||
return await self._create_token()
|
||||
else:
|
||||
return await self._create_token()
|
||||
|
||||
async def _create_token(self) -> str:
|
||||
"""Create a new GoCardless access token"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/token/new/",
|
||||
json={
|
||||
"secret_id": self.config["key"],
|
||||
"secret_key": self.config["secret"],
|
||||
},
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
auth = response.json()
|
||||
self._save_auth(auth)
|
||||
self._token = auth["access"]
|
||||
return self._token
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create GoCardless token: {e}")
|
||||
raise
|
||||
|
||||
def _save_auth(self, auth_data: dict):
|
||||
"""Save authentication data to file"""
|
||||
auth_file = Path.home() / ".config" / "leggen" / "auth.json"
|
||||
auth_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(auth_file, "w") as f:
|
||||
json.dump(auth_data, f)
|
||||
|
||||
async def get_institutions(self, country: str = "PT") -> List[Dict[str, Any]]:
|
||||
"""Get available bank institutions for a country"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(
|
||||
f"{self.base_url}/institutions/",
|
||||
headers=headers,
|
||||
params={"country": country},
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def create_requisition(
|
||||
self, institution_id: str, redirect_url: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Create a bank connection requisition"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/requisitions/",
|
||||
headers=headers,
|
||||
json={"institution_id": institution_id, "redirect": redirect_url},
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def get_requisitions(self) -> Dict[str, Any]:
|
||||
"""Get all requisitions"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(
|
||||
f"{self.base_url}/requisitions/", headers=headers
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def get_account_details(self, account_id: str) -> Dict[str, Any]:
|
||||
"""Get account details"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(
|
||||
f"{self.base_url}/accounts/{account_id}/", headers=headers
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def get_account_balances(self, account_id: str) -> Dict[str, Any]:
|
||||
"""Get account balances"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(
|
||||
f"{self.base_url}/accounts/{account_id}/balances/", headers=headers
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def get_account_transactions(self, account_id: str) -> Dict[str, Any]:
|
||||
"""Get account transactions"""
|
||||
headers = await self._get_auth_headers()
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(
|
||||
f"{self.base_url}/accounts/{account_id}/transactions/", headers=headers
|
||||
)
|
||||
_log_rate_limits(response)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
201
leggen/services/notification_service.py
Normal file
201
leggen/services/notification_service.py
Normal file
@@ -0,0 +1,201 @@
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from leggen.utils.config import config
|
||||
|
||||
|
||||
class NotificationService:
|
||||
def __init__(self):
|
||||
self.notifications_config = config.notifications_config
|
||||
self.filters_config = config.filters_config
|
||||
|
||||
async def send_transaction_notifications(
|
||||
self, transactions: List[Dict[str, Any]]
|
||||
) -> None:
|
||||
"""Send notifications for new transactions that match filters"""
|
||||
if not self.filters_config:
|
||||
logger.info("No notification filters configured, skipping notifications")
|
||||
return
|
||||
|
||||
# Filter transactions that match notification criteria
|
||||
matching_transactions = self._filter_transactions(transactions)
|
||||
|
||||
if not matching_transactions:
|
||||
logger.info("No transactions matched notification filters")
|
||||
return
|
||||
|
||||
# Send to enabled notification services
|
||||
if self._is_discord_enabled():
|
||||
await self._send_discord_notifications(matching_transactions)
|
||||
|
||||
if self._is_telegram_enabled():
|
||||
await self._send_telegram_notifications(matching_transactions)
|
||||
|
||||
async def send_test_notification(self, service: str, message: str) -> bool:
|
||||
"""Send a test notification"""
|
||||
try:
|
||||
if service == "discord" and self._is_discord_enabled():
|
||||
await self._send_discord_test(message)
|
||||
return True
|
||||
elif service == "telegram" and self._is_telegram_enabled():
|
||||
await self._send_telegram_test(message)
|
||||
return True
|
||||
else:
|
||||
logger.error(
|
||||
f"Notification service '{service}' not enabled or not found"
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send test notification to {service}: {e}")
|
||||
return False
|
||||
|
||||
async def send_expiry_notification(self, notification_data: Dict[str, Any]) -> None:
|
||||
"""Send notification about account expiry"""
|
||||
if self._is_discord_enabled():
|
||||
await self._send_discord_expiry(notification_data)
|
||||
|
||||
if self._is_telegram_enabled():
|
||||
await self._send_telegram_expiry(notification_data)
|
||||
|
||||
def _filter_transactions(
|
||||
self, transactions: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Filter transactions based on notification criteria"""
|
||||
matching = []
|
||||
filters_case_insensitive = self.filters_config.get("case-insensitive", [])
|
||||
filters_case_sensitive = self.filters_config.get("case-sensitive", [])
|
||||
|
||||
for transaction in transactions:
|
||||
description = transaction.get("description", "")
|
||||
description_lower = description.lower()
|
||||
|
||||
# Check case-insensitive filters
|
||||
for filter_value in filters_case_insensitive:
|
||||
if filter_value.lower() in description_lower:
|
||||
matching.append(
|
||||
{
|
||||
"name": transaction["description"],
|
||||
"value": transaction["transactionValue"],
|
||||
"currency": transaction["transactionCurrency"],
|
||||
"date": transaction["transactionDate"],
|
||||
}
|
||||
)
|
||||
break
|
||||
|
||||
# Check case-sensitive filters
|
||||
for filter_value in filters_case_sensitive:
|
||||
if filter_value in description:
|
||||
matching.append(
|
||||
{
|
||||
"name": transaction["description"],
|
||||
"value": transaction["transactionValue"],
|
||||
"currency": transaction["transactionCurrency"],
|
||||
"date": transaction["transactionDate"],
|
||||
}
|
||||
)
|
||||
break
|
||||
|
||||
return matching
|
||||
|
||||
def _is_discord_enabled(self) -> bool:
|
||||
"""Check if Discord notifications are enabled"""
|
||||
discord_config = self.notifications_config.get("discord", {})
|
||||
return bool(
|
||||
discord_config.get("webhook") and discord_config.get("enabled", True)
|
||||
)
|
||||
|
||||
def _is_telegram_enabled(self) -> bool:
|
||||
"""Check if Telegram notifications are enabled"""
|
||||
telegram_config = self.notifications_config.get("telegram", {})
|
||||
return bool(
|
||||
telegram_config.get("api-key")
|
||||
and telegram_config.get("chat-id")
|
||||
and telegram_config.get("enabled", True)
|
||||
)
|
||||
|
||||
async def _send_discord_notifications(
|
||||
self, transactions: List[Dict[str, Any]]
|
||||
) -> None:
|
||||
"""Send Discord notifications - placeholder implementation"""
|
||||
# Would import and use leggen.notifications.discord
|
||||
logger.info(f"Sending {len(transactions)} transaction notifications to Discord")
|
||||
|
||||
async def _send_telegram_notifications(
|
||||
self, transactions: List[Dict[str, Any]]
|
||||
) -> None:
|
||||
"""Send Telegram notifications - placeholder implementation"""
|
||||
# Would import and use leggen.notifications.telegram
|
||||
logger.info(
|
||||
f"Sending {len(transactions)} transaction notifications to Telegram"
|
||||
)
|
||||
|
||||
async def _send_discord_test(self, message: str) -> None:
|
||||
"""Send Discord test notification"""
|
||||
try:
|
||||
from leggen.notifications.discord import send_expire_notification
|
||||
import click
|
||||
|
||||
# Create a mock context with the webhook
|
||||
ctx = click.Context(click.Command("test"))
|
||||
ctx.obj = {
|
||||
"notifications": {
|
||||
"discord": {
|
||||
"webhook": self.notifications_config.get("discord", {}).get(
|
||||
"webhook"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Send test notification using the actual implementation
|
||||
test_notification = {
|
||||
"bank": "Test",
|
||||
"requisition_id": "test-123",
|
||||
"status": "active",
|
||||
"days_left": 30,
|
||||
}
|
||||
send_expire_notification(ctx, test_notification)
|
||||
logger.info(f"Discord test notification sent: {message}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Discord test notification: {e}")
|
||||
raise
|
||||
|
||||
async def _send_telegram_test(self, message: str) -> None:
|
||||
"""Send Telegram test notification"""
|
||||
try:
|
||||
from leggen.notifications.telegram import send_expire_notification
|
||||
import click
|
||||
|
||||
# Create a mock context with the telegram config
|
||||
ctx = click.Context(click.Command("test"))
|
||||
telegram_config = self.notifications_config.get("telegram", {})
|
||||
ctx.obj = {
|
||||
"notifications": {
|
||||
"telegram": {
|
||||
"api-key": telegram_config.get("api-key"),
|
||||
"chat-id": telegram_config.get("chat-id"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Send test notification using the actual implementation
|
||||
test_notification = {
|
||||
"bank": "Test",
|
||||
"requisition_id": "test-123",
|
||||
"status": "active",
|
||||
"days_left": 30,
|
||||
}
|
||||
send_expire_notification(ctx, test_notification)
|
||||
logger.info(f"Telegram test notification sent: {message}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Telegram test notification: {e}")
|
||||
raise
|
||||
|
||||
async def _send_discord_expiry(self, notification_data: Dict[str, Any]) -> None:
|
||||
"""Send Discord expiry notification"""
|
||||
logger.info(f"Sending Discord expiry notification: {notification_data}")
|
||||
|
||||
async def _send_telegram_expiry(self, notification_data: Dict[str, Any]) -> None:
|
||||
"""Send Telegram expiry notification"""
|
||||
logger.info(f"Sending Telegram expiry notification: {notification_data}")
|
||||
187
leggen/services/sync_service.py
Normal file
187
leggen/services/sync_service.py
Normal file
@@ -0,0 +1,187 @@
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from leggen.api.models.sync import SyncResult, SyncStatus
|
||||
from leggen.services.gocardless_service import GoCardlessService
|
||||
from leggen.services.database_service import DatabaseService
|
||||
from leggen.services.notification_service import NotificationService
|
||||
|
||||
|
||||
class SyncService:
|
||||
def __init__(self):
|
||||
self.gocardless = GoCardlessService()
|
||||
self.database = DatabaseService()
|
||||
self.notifications = NotificationService()
|
||||
self._sync_status = SyncStatus(is_running=False)
|
||||
|
||||
async def get_sync_status(self) -> SyncStatus:
|
||||
"""Get current sync status"""
|
||||
return self._sync_status
|
||||
|
||||
async def sync_all_accounts(self, force: bool = False) -> SyncResult:
|
||||
"""Sync all connected accounts"""
|
||||
if self._sync_status.is_running and not force:
|
||||
raise Exception("Sync is already running")
|
||||
|
||||
start_time = datetime.now()
|
||||
self._sync_status.is_running = True
|
||||
self._sync_status.errors = []
|
||||
|
||||
accounts_processed = 0
|
||||
transactions_added = 0
|
||||
transactions_updated = 0
|
||||
balances_updated = 0
|
||||
errors = []
|
||||
|
||||
try:
|
||||
logger.info("Starting sync of all accounts")
|
||||
|
||||
# Get all requisitions and accounts
|
||||
requisitions = await self.gocardless.get_requisitions()
|
||||
all_accounts = set()
|
||||
|
||||
for req in requisitions.get("results", []):
|
||||
all_accounts.update(req.get("accounts", []))
|
||||
|
||||
self._sync_status.total_accounts = len(all_accounts)
|
||||
|
||||
# Process each account
|
||||
for account_id in all_accounts:
|
||||
try:
|
||||
# Get account details
|
||||
account_details = await self.gocardless.get_account_details(
|
||||
account_id
|
||||
)
|
||||
|
||||
# Get balances to extract currency information
|
||||
balances = await self.gocardless.get_account_balances(account_id)
|
||||
|
||||
# Enrich account details with currency and persist
|
||||
if account_details and balances:
|
||||
enriched_account_details = account_details.copy()
|
||||
|
||||
# Extract currency from first balance
|
||||
balances_list = balances.get("balances", [])
|
||||
if balances_list:
|
||||
first_balance = balances_list[0]
|
||||
balance_amount = first_balance.get("balanceAmount", {})
|
||||
currency = balance_amount.get("currency")
|
||||
if currency:
|
||||
enriched_account_details["currency"] = currency
|
||||
|
||||
# Persist enriched account details to database
|
||||
await self.database.persist_account_details(
|
||||
enriched_account_details
|
||||
)
|
||||
|
||||
# Merge account details into balances data for proper persistence
|
||||
balances_with_account_info = balances.copy()
|
||||
balances_with_account_info["institution_id"] = (
|
||||
enriched_account_details.get("institution_id")
|
||||
)
|
||||
balances_with_account_info["iban"] = (
|
||||
enriched_account_details.get("iban")
|
||||
)
|
||||
balances_with_account_info["account_status"] = (
|
||||
enriched_account_details.get("status")
|
||||
)
|
||||
await self.database.persist_balance(
|
||||
account_id, balances_with_account_info
|
||||
)
|
||||
balances_updated += len(balances.get("balances", []))
|
||||
elif account_details:
|
||||
# Fallback: persist account details without currency if balances failed
|
||||
await self.database.persist_account_details(account_details)
|
||||
|
||||
# Get and save transactions
|
||||
transactions = await self.gocardless.get_account_transactions(
|
||||
account_id
|
||||
)
|
||||
if transactions:
|
||||
processed_transactions = self.database.process_transactions(
|
||||
account_id, account_details, transactions
|
||||
)
|
||||
new_transactions = await self.database.persist_transactions(
|
||||
account_id, processed_transactions
|
||||
)
|
||||
transactions_added += len(new_transactions)
|
||||
|
||||
# Send notifications for new transactions
|
||||
if new_transactions:
|
||||
await self.notifications.send_transaction_notifications(
|
||||
new_transactions
|
||||
)
|
||||
|
||||
accounts_processed += 1
|
||||
self._sync_status.accounts_synced = accounts_processed
|
||||
|
||||
logger.info(f"Synced account {account_id} successfully")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to sync account {account_id}: {str(e)}"
|
||||
errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
end_time = datetime.now()
|
||||
duration = (end_time - start_time).total_seconds()
|
||||
|
||||
self._sync_status.last_sync = end_time
|
||||
|
||||
result = SyncResult(
|
||||
success=len(errors) == 0,
|
||||
accounts_processed=accounts_processed,
|
||||
transactions_added=transactions_added,
|
||||
transactions_updated=transactions_updated,
|
||||
balances_updated=balances_updated,
|
||||
duration_seconds=duration,
|
||||
errors=errors,
|
||||
started_at=start_time,
|
||||
completed_at=end_time,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Sync completed: {accounts_processed} accounts, {transactions_added} new transactions"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Sync failed: {str(e)}"
|
||||
errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
raise
|
||||
finally:
|
||||
self._sync_status.is_running = False
|
||||
|
||||
async def sync_specific_accounts(
|
||||
self, account_ids: List[str], force: bool = False
|
||||
) -> SyncResult:
|
||||
"""Sync specific accounts"""
|
||||
if self._sync_status.is_running and not force:
|
||||
raise Exception("Sync is already running")
|
||||
|
||||
# Similar implementation but only for specified accounts
|
||||
# For brevity, implementing a simplified version
|
||||
start_time = datetime.now()
|
||||
self._sync_status.is_running = True
|
||||
|
||||
try:
|
||||
# Process only specified accounts
|
||||
# Implementation would be similar to sync_all_accounts
|
||||
# but filtered to only the specified account_ids
|
||||
|
||||
end_time = datetime.now()
|
||||
return SyncResult(
|
||||
success=True,
|
||||
accounts_processed=len(account_ids),
|
||||
transactions_added=0,
|
||||
transactions_updated=0,
|
||||
balances_updated=0,
|
||||
duration_seconds=(end_time - start_time).total_seconds(),
|
||||
errors=[],
|
||||
started_at=start_time,
|
||||
completed_at=end_time,
|
||||
)
|
||||
finally:
|
||||
self._sync_status.is_running = False
|
||||
Reference in New Issue
Block a user