mirror of
https://github.com/elisiariocouto/leggen.git
synced 2025-12-16 01:42:19 +00:00
refactor(api): Improve database connection management and reduce boilerplate.
- Add context manager for database connections with proper cleanup - Add @require_sqlite decorator to eliminate duplicate checks - Refactor 9 core CRUD methods to use managed connections - Reduce code by 50 lines while improving resource management - All 114 tests passing
This commit is contained in:
@@ -1,6 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
from contextlib import contextmanager
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from functools import wraps
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
@@ -14,6 +16,25 @@ from leggen.utils.config import config
|
|||||||
from leggen.utils.paths import path_manager
|
from leggen.utils.paths import path_manager
|
||||||
|
|
||||||
|
|
||||||
|
def require_sqlite(func):
|
||||||
|
"""Decorator to check if SQLite is enabled before executing method"""
|
||||||
|
|
||||||
|
@wraps(func)
|
||||||
|
async def wrapper(self, *args, **kwargs):
|
||||||
|
if not self.sqlite_enabled:
|
||||||
|
logger.warning(f"SQLite database disabled, skipping {func.__name__}")
|
||||||
|
# Return appropriate default based on return type hints
|
||||||
|
return_type = func.__annotations__.get("return")
|
||||||
|
if return_type is int:
|
||||||
|
return 0
|
||||||
|
elif return_type in (list, List[Dict[str, Any]]):
|
||||||
|
return []
|
||||||
|
return None
|
||||||
|
return await func(self, *args, **kwargs)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
class DatabaseService:
|
class DatabaseService:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.db_config = config.database_config
|
self.db_config = config.database_config
|
||||||
@@ -24,24 +45,33 @@ class DatabaseService:
|
|||||||
self.balance_transformer = BalanceTransformer()
|
self.balance_transformer = BalanceTransformer()
|
||||||
self.analytics_processor = AnalyticsProcessor()
|
self.analytics_processor = AnalyticsProcessor()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _get_db_connection(self, row_factory: bool = False):
|
||||||
|
"""Context manager for database connections with proper cleanup"""
|
||||||
|
db_path = path_manager.get_database_path()
|
||||||
|
conn = sqlite3.connect(str(db_path))
|
||||||
|
if row_factory:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
yield conn
|
||||||
|
except Exception as e:
|
||||||
|
conn.rollback()
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def persist_balance(
|
async def persist_balance(
|
||||||
self, account_id: str, balance_data: Dict[str, Any]
|
self, account_id: str, balance_data: Dict[str, Any]
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Persist account balance data"""
|
"""Persist account balance data"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, skipping balance persistence")
|
|
||||||
return
|
|
||||||
|
|
||||||
await self._persist_balance_sqlite(account_id, balance_data)
|
await self._persist_balance_sqlite(account_id, balance_data)
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def persist_transactions(
|
async def persist_transactions(
|
||||||
self, account_id: str, transactions: List[Dict[str, Any]]
|
self, account_id: str, transactions: List[Dict[str, Any]]
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Persist transactions and return new transactions"""
|
"""Persist transactions and return new transactions"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, skipping transaction persistence")
|
|
||||||
return transactions
|
|
||||||
|
|
||||||
return await self._persist_transactions_sqlite(account_id, transactions)
|
return await self._persist_transactions_sqlite(account_id, transactions)
|
||||||
|
|
||||||
def process_transactions(
|
def process_transactions(
|
||||||
@@ -55,6 +85,7 @@ class DatabaseService:
|
|||||||
account_id, account_info, transaction_data
|
account_id, account_info, transaction_data
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_transactions_from_db(
|
async def get_transactions_from_db(
|
||||||
self,
|
self,
|
||||||
account_id: Optional[str] = None,
|
account_id: Optional[str] = None,
|
||||||
@@ -67,10 +98,6 @@ class DatabaseService:
|
|||||||
search: Optional[str] = None,
|
search: Optional[str] = None,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Get transactions from SQLite database"""
|
"""Get transactions from SQLite database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read transactions")
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
transactions = self._get_transactions(
|
transactions = self._get_transactions(
|
||||||
account_id=account_id,
|
account_id=account_id,
|
||||||
@@ -88,6 +115,7 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get transactions from database: {e}")
|
logger.error(f"Failed to get transactions from database: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_transaction_count_from_db(
|
async def get_transaction_count_from_db(
|
||||||
self,
|
self,
|
||||||
account_id: Optional[str] = None,
|
account_id: Optional[str] = None,
|
||||||
@@ -98,9 +126,6 @@ class DatabaseService:
|
|||||||
search: Optional[str] = None,
|
search: Optional[str] = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Get total count of transactions from SQLite database"""
|
"""Get total count of transactions from SQLite database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
filters = {
|
filters = {
|
||||||
"date_from": date_from,
|
"date_from": date_from,
|
||||||
@@ -119,14 +144,11 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get transaction count from database: {e}")
|
logger.error(f"Failed to get transaction count from database: {e}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_balances_from_db(
|
async def get_balances_from_db(
|
||||||
self, account_id: Optional[str] = None
|
self, account_id: Optional[str] = None
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Get balances from SQLite database"""
|
"""Get balances from SQLite database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read balances")
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
balances = self._get_balances(account_id=account_id)
|
balances = self._get_balances(account_id=account_id)
|
||||||
logger.debug(f"Retrieved {len(balances)} balances from database")
|
logger.debug(f"Retrieved {len(balances)} balances from database")
|
||||||
@@ -135,14 +157,11 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get balances from database: {e}")
|
logger.error(f"Failed to get balances from database: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_historical_balances_from_db(
|
async def get_historical_balances_from_db(
|
||||||
self, account_id: Optional[str] = None, days: int = 365
|
self, account_id: Optional[str] = None, days: int = 365
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Get historical balance progression from SQLite database"""
|
"""Get historical balance progression from SQLite database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read historical balances")
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
balances = self.analytics_processor.calculate_historical_balances(
|
balances = self.analytics_processor.calculate_historical_balances(
|
||||||
@@ -156,13 +175,11 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get historical balances from database: {e}")
|
logger.error(f"Failed to get historical balances from database: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_account_summary_from_db(
|
async def get_account_summary_from_db(
|
||||||
self, account_id: str
|
self, account_id: str
|
||||||
) -> Optional[Dict[str, Any]]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
"""Get basic account info from SQLite database (avoids GoCardless call)"""
|
"""Get basic account info from SQLite database (avoids GoCardless call)"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
summary = self._get_account_summary(account_id)
|
summary = self._get_account_summary(account_id)
|
||||||
if summary:
|
if summary:
|
||||||
@@ -174,22 +191,16 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get account summary from database: {e}")
|
logger.error(f"Failed to get account summary from database: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def persist_account_details(self, account_data: Dict[str, Any]) -> None:
|
async def persist_account_details(self, account_data: Dict[str, Any]) -> None:
|
||||||
"""Persist account details to database"""
|
"""Persist account details to database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, skipping account persistence")
|
|
||||||
return
|
|
||||||
|
|
||||||
await self._persist_account_details_sqlite(account_data)
|
await self._persist_account_details_sqlite(account_data)
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_accounts_from_db(
|
async def get_accounts_from_db(
|
||||||
self, account_ids: Optional[List[str]] = None
|
self, account_ids: Optional[List[str]] = None
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Get account details from database"""
|
"""Get account details from database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read accounts")
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
accounts = self._get_accounts(account_ids=account_ids)
|
accounts = self._get_accounts(account_ids=account_ids)
|
||||||
logger.debug(f"Retrieved {len(accounts)} accounts from database")
|
logger.debug(f"Retrieved {len(accounts)} accounts from database")
|
||||||
@@ -198,14 +209,11 @@ class DatabaseService:
|
|||||||
logger.error(f"Failed to get accounts from database: {e}")
|
logger.error(f"Failed to get accounts from database: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@require_sqlite
|
||||||
async def get_account_details_from_db(
|
async def get_account_details_from_db(
|
||||||
self, account_id: str
|
self, account_id: str
|
||||||
) -> Optional[Dict[str, Any]]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
"""Get specific account details from database"""
|
"""Get specific account details from database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read account")
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
account = self._get_account(account_id)
|
account = self._get_account(account_id)
|
||||||
if account:
|
if account:
|
||||||
@@ -729,66 +737,62 @@ class DatabaseService:
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""Persist balance to SQLite"""
|
"""Persist balance to SQLite"""
|
||||||
try:
|
try:
|
||||||
import sqlite3
|
with self._get_db_connection() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
db_path = path_manager.get_database_path()
|
# Create the balances table if it doesn't exist
|
||||||
conn = sqlite3.connect(str(db_path))
|
cursor.execute(
|
||||||
cursor = conn.cursor()
|
"""CREATE TABLE IF NOT EXISTS balances (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
account_id TEXT,
|
||||||
|
bank TEXT,
|
||||||
|
status TEXT,
|
||||||
|
iban TEXT,
|
||||||
|
amount REAL,
|
||||||
|
currency TEXT,
|
||||||
|
type TEXT,
|
||||||
|
timestamp DATETIME
|
||||||
|
)"""
|
||||||
|
)
|
||||||
|
|
||||||
# Create the balances table if it doesn't exist
|
# Create indexes for better performance
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""CREATE TABLE IF NOT EXISTS balances (
|
"""CREATE INDEX IF NOT EXISTS idx_balances_account_id
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
ON balances(account_id)"""
|
||||||
account_id TEXT,
|
)
|
||||||
bank TEXT,
|
cursor.execute(
|
||||||
status TEXT,
|
"""CREATE INDEX IF NOT EXISTS idx_balances_timestamp
|
||||||
iban TEXT,
|
ON balances(timestamp)"""
|
||||||
amount REAL,
|
)
|
||||||
currency TEXT,
|
cursor.execute(
|
||||||
type TEXT,
|
"""CREATE INDEX IF NOT EXISTS idx_balances_account_type_timestamp
|
||||||
timestamp DATETIME
|
ON balances(account_id, type, timestamp)"""
|
||||||
)"""
|
)
|
||||||
)
|
|
||||||
|
|
||||||
# Create indexes for better performance
|
# Transform and persist balances
|
||||||
cursor.execute(
|
balance_rows = self.balance_transformer.transform_to_database_format(
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_balances_account_id
|
account_id, balance_data
|
||||||
ON balances(account_id)"""
|
)
|
||||||
)
|
|
||||||
cursor.execute(
|
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_balances_timestamp
|
|
||||||
ON balances(timestamp)"""
|
|
||||||
)
|
|
||||||
cursor.execute(
|
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_balances_account_type_timestamp
|
|
||||||
ON balances(account_id, type, timestamp)"""
|
|
||||||
)
|
|
||||||
|
|
||||||
# Transform and persist balances
|
for row in balance_rows:
|
||||||
balance_rows = self.balance_transformer.transform_to_database_format(
|
try:
|
||||||
account_id, balance_data
|
cursor.execute(
|
||||||
)
|
"""INSERT INTO balances (
|
||||||
|
account_id,
|
||||||
|
bank,
|
||||||
|
status,
|
||||||
|
iban,
|
||||||
|
amount,
|
||||||
|
currency,
|
||||||
|
type,
|
||||||
|
timestamp
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
row,
|
||||||
|
)
|
||||||
|
except sqlite3.IntegrityError:
|
||||||
|
logger.warning(f"Skipped duplicate balance for {account_id}")
|
||||||
|
|
||||||
for row in balance_rows:
|
conn.commit()
|
||||||
try:
|
|
||||||
cursor.execute(
|
|
||||||
"""INSERT INTO balances (
|
|
||||||
account_id,
|
|
||||||
bank,
|
|
||||||
status,
|
|
||||||
iban,
|
|
||||||
amount,
|
|
||||||
currency,
|
|
||||||
type,
|
|
||||||
timestamp
|
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
||||||
row,
|
|
||||||
)
|
|
||||||
except sqlite3.IntegrityError:
|
|
||||||
logger.warning(f"Skipped duplicate balance for {account_id}")
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
logger.info(f"Persisted balances to SQLite for account {account_id}")
|
logger.info(f"Persisted balances to SQLite for account {account_id}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -800,106 +804,101 @@ class DatabaseService:
|
|||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Persist transactions to SQLite"""
|
"""Persist transactions to SQLite"""
|
||||||
try:
|
try:
|
||||||
import json
|
with self._get_db_connection() as conn:
|
||||||
import sqlite3
|
cursor = conn.cursor()
|
||||||
|
|
||||||
db_path = path_manager.get_database_path()
|
# The table should already exist with the new schema from migration
|
||||||
conn = sqlite3.connect(str(db_path))
|
# If it doesn't exist, create it (for new installations)
|
||||||
cursor = conn.cursor()
|
cursor.execute(
|
||||||
|
"""CREATE TABLE IF NOT EXISTS transactions (
|
||||||
|
accountId TEXT NOT NULL,
|
||||||
|
transactionId TEXT NOT NULL,
|
||||||
|
internalTransactionId TEXT,
|
||||||
|
institutionId TEXT,
|
||||||
|
iban TEXT,
|
||||||
|
transactionDate DATETIME,
|
||||||
|
description TEXT,
|
||||||
|
transactionValue REAL,
|
||||||
|
transactionCurrency TEXT,
|
||||||
|
transactionStatus TEXT,
|
||||||
|
rawTransaction JSON,
|
||||||
|
PRIMARY KEY (accountId, transactionId)
|
||||||
|
)"""
|
||||||
|
)
|
||||||
|
|
||||||
# The table should already exist with the new schema from migration
|
# Create indexes for better performance (if they don't exist)
|
||||||
# If it doesn't exist, create it (for new installations)
|
cursor.execute(
|
||||||
cursor.execute(
|
"""CREATE INDEX IF NOT EXISTS idx_transactions_internal_id
|
||||||
"""CREATE TABLE IF NOT EXISTS transactions (
|
ON transactions(internalTransactionId)"""
|
||||||
accountId TEXT NOT NULL,
|
)
|
||||||
transactionId TEXT NOT NULL,
|
cursor.execute(
|
||||||
internalTransactionId TEXT,
|
"""CREATE INDEX IF NOT EXISTS idx_transactions_date
|
||||||
institutionId TEXT,
|
ON transactions(transactionDate)"""
|
||||||
iban TEXT,
|
)
|
||||||
transactionDate DATETIME,
|
cursor.execute(
|
||||||
description TEXT,
|
"""CREATE INDEX IF NOT EXISTS idx_transactions_account_date
|
||||||
transactionValue REAL,
|
ON transactions(accountId, transactionDate)"""
|
||||||
transactionCurrency TEXT,
|
)
|
||||||
transactionStatus TEXT,
|
cursor.execute(
|
||||||
rawTransaction JSON,
|
"""CREATE INDEX IF NOT EXISTS idx_transactions_amount
|
||||||
PRIMARY KEY (accountId, transactionId)
|
ON transactions(transactionValue)"""
|
||||||
)"""
|
)
|
||||||
)
|
|
||||||
|
|
||||||
# Create indexes for better performance (if they don't exist)
|
# Prepare an SQL statement for inserting/replacing data
|
||||||
cursor.execute(
|
insert_sql = """INSERT OR REPLACE INTO transactions (
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_internal_id
|
accountId,
|
||||||
ON transactions(internalTransactionId)"""
|
transactionId,
|
||||||
)
|
internalTransactionId,
|
||||||
cursor.execute(
|
institutionId,
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_date
|
iban,
|
||||||
ON transactions(transactionDate)"""
|
transactionDate,
|
||||||
)
|
description,
|
||||||
cursor.execute(
|
transactionValue,
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_account_date
|
transactionCurrency,
|
||||||
ON transactions(accountId, transactionDate)"""
|
transactionStatus,
|
||||||
)
|
rawTransaction
|
||||||
cursor.execute(
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_transactions_amount
|
|
||||||
ON transactions(transactionValue)"""
|
|
||||||
)
|
|
||||||
|
|
||||||
# Prepare an SQL statement for inserting/replacing data
|
new_transactions = []
|
||||||
insert_sql = """INSERT OR REPLACE INTO transactions (
|
|
||||||
accountId,
|
|
||||||
transactionId,
|
|
||||||
internalTransactionId,
|
|
||||||
institutionId,
|
|
||||||
iban,
|
|
||||||
transactionDate,
|
|
||||||
description,
|
|
||||||
transactionValue,
|
|
||||||
transactionCurrency,
|
|
||||||
transactionStatus,
|
|
||||||
rawTransaction
|
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
|
|
||||||
|
|
||||||
new_transactions = []
|
for transaction in transactions:
|
||||||
|
try:
|
||||||
|
# Check if transaction already exists before insertion
|
||||||
|
cursor.execute(
|
||||||
|
"""SELECT COUNT(*) FROM transactions
|
||||||
|
WHERE accountId = ? AND transactionId = ?""",
|
||||||
|
(transaction["accountId"], transaction["transactionId"]),
|
||||||
|
)
|
||||||
|
exists = cursor.fetchone()[0] > 0
|
||||||
|
|
||||||
for transaction in transactions:
|
cursor.execute(
|
||||||
try:
|
insert_sql,
|
||||||
# Check if transaction already exists before insertion
|
(
|
||||||
cursor.execute(
|
transaction["accountId"],
|
||||||
"""SELECT COUNT(*) FROM transactions
|
transaction["transactionId"],
|
||||||
WHERE accountId = ? AND transactionId = ?""",
|
transaction.get("internalTransactionId"),
|
||||||
(transaction["accountId"], transaction["transactionId"]),
|
transaction["institutionId"],
|
||||||
)
|
transaction["iban"],
|
||||||
exists = cursor.fetchone()[0] > 0
|
transaction["transactionDate"],
|
||||||
|
transaction["description"],
|
||||||
|
transaction["transactionValue"],
|
||||||
|
transaction["transactionCurrency"],
|
||||||
|
transaction["transactionStatus"],
|
||||||
|
json.dumps(transaction["rawTransaction"]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
cursor.execute(
|
# Only add to new_transactions if it didn't exist before
|
||||||
insert_sql,
|
if not exists:
|
||||||
(
|
new_transactions.append(transaction)
|
||||||
transaction["accountId"],
|
|
||||||
transaction["transactionId"],
|
|
||||||
transaction.get("internalTransactionId"),
|
|
||||||
transaction["institutionId"],
|
|
||||||
transaction["iban"],
|
|
||||||
transaction["transactionDate"],
|
|
||||||
transaction["description"],
|
|
||||||
transaction["transactionValue"],
|
|
||||||
transaction["transactionCurrency"],
|
|
||||||
transaction["transactionStatus"],
|
|
||||||
json.dumps(transaction["rawTransaction"]),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Only add to new_transactions if it didn't exist before
|
except sqlite3.IntegrityError as e:
|
||||||
if not exists:
|
logger.warning(
|
||||||
new_transactions.append(transaction)
|
f"Failed to insert transaction {transaction.get('transactionId')}: {e}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
except sqlite3.IntegrityError as e:
|
conn.commit()
|
||||||
logger.warning(
|
|
||||||
f"Failed to insert transaction {transaction.get('transactionId')}: {e}"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Persisted {len(new_transactions)} new transactions to SQLite for account {account_id}"
|
f"Persisted {len(new_transactions)} new transactions to SQLite for account {account_id}"
|
||||||
@@ -939,50 +938,49 @@ class DatabaseService:
|
|||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return []
|
return []
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
conn.row_factory = sqlite3.Row # Enable dict-like access
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
# Build query with filters
|
with self._get_db_connection(row_factory=True) as conn:
|
||||||
query = "SELECT * FROM transactions WHERE 1=1"
|
cursor = conn.cursor()
|
||||||
params = []
|
|
||||||
|
|
||||||
if account_id:
|
# Build query with filters
|
||||||
query += " AND accountId = ?"
|
query = "SELECT * FROM transactions WHERE 1=1"
|
||||||
params.append(account_id)
|
params = []
|
||||||
|
|
||||||
if date_from:
|
if account_id:
|
||||||
query += " AND transactionDate >= ?"
|
query += " AND accountId = ?"
|
||||||
params.append(date_from)
|
params.append(account_id)
|
||||||
|
|
||||||
if date_to:
|
if date_from:
|
||||||
query += " AND transactionDate <= ?"
|
query += " AND transactionDate >= ?"
|
||||||
params.append(date_to)
|
params.append(date_from)
|
||||||
|
|
||||||
if min_amount is not None:
|
if date_to:
|
||||||
query += " AND transactionValue >= ?"
|
query += " AND transactionDate <= ?"
|
||||||
params.append(min_amount)
|
params.append(date_to)
|
||||||
|
|
||||||
if max_amount is not None:
|
if min_amount is not None:
|
||||||
query += " AND transactionValue <= ?"
|
query += " AND transactionValue >= ?"
|
||||||
params.append(max_amount)
|
params.append(min_amount)
|
||||||
|
|
||||||
if search:
|
if max_amount is not None:
|
||||||
query += " AND description LIKE ?"
|
query += " AND transactionValue <= ?"
|
||||||
params.append(f"%{search}%")
|
params.append(max_amount)
|
||||||
|
|
||||||
# Add ordering and pagination
|
if search:
|
||||||
query += " ORDER BY transactionDate DESC"
|
query += " AND description LIKE ?"
|
||||||
|
params.append(f"%{search}%")
|
||||||
|
|
||||||
if limit:
|
# Add ordering and pagination
|
||||||
query += " LIMIT ?"
|
query += " ORDER BY transactionDate DESC"
|
||||||
params.append(limit)
|
|
||||||
|
|
||||||
if offset:
|
if limit:
|
||||||
query += " OFFSET ?"
|
query += " LIMIT ?"
|
||||||
params.append(offset)
|
params.append(limit)
|
||||||
|
|
||||||
|
if offset:
|
||||||
|
query += " OFFSET ?"
|
||||||
|
params.append(offset)
|
||||||
|
|
||||||
try:
|
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
|
|
||||||
@@ -996,61 +994,48 @@ class DatabaseService:
|
|||||||
)
|
)
|
||||||
transactions.append(transaction)
|
transactions.append(transaction)
|
||||||
|
|
||||||
conn.close()
|
|
||||||
return transactions
|
return transactions
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _get_balances(self, account_id=None):
|
def _get_balances(self, account_id=None):
|
||||||
"""Get latest balances from SQLite database"""
|
"""Get latest balances from SQLite database"""
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return []
|
return []
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
# Get latest balance for each account_id and type combination
|
with self._get_db_connection(row_factory=True) as conn:
|
||||||
query = """
|
cursor = conn.cursor()
|
||||||
SELECT * FROM balances b1
|
|
||||||
WHERE b1.timestamp = (
|
|
||||||
SELECT MAX(b2.timestamp)
|
|
||||||
FROM balances b2
|
|
||||||
WHERE b2.account_id = b1.account_id AND b2.type = b1.type
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
params = []
|
|
||||||
|
|
||||||
if account_id:
|
# Get latest balance for each account_id and type combination
|
||||||
query += " AND b1.account_id = ?"
|
query = """
|
||||||
params.append(account_id)
|
SELECT * FROM balances b1
|
||||||
|
WHERE b1.timestamp = (
|
||||||
|
SELECT MAX(b2.timestamp)
|
||||||
|
FROM balances b2
|
||||||
|
WHERE b2.account_id = b1.account_id AND b2.type = b1.type
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
params = []
|
||||||
|
|
||||||
query += " ORDER BY b1.account_id, b1.type"
|
if account_id:
|
||||||
|
query += " AND b1.account_id = ?"
|
||||||
|
params.append(account_id)
|
||||||
|
|
||||||
|
query += " ORDER BY b1.account_id, b1.type"
|
||||||
|
|
||||||
try:
|
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
|
|
||||||
balances = [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
conn.close()
|
|
||||||
return balances
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _get_account_summary(self, account_id):
|
def _get_account_summary(self, account_id):
|
||||||
"""Get basic account info from transactions table (avoids GoCardless API call)"""
|
"""Get basic account info from transactions table (avoids GoCardless API call)"""
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return None
|
return None
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
try:
|
with self._get_db_connection(row_factory=True) as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
# Get account info from most recent transaction
|
# Get account info from most recent transaction
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""
|
"""
|
||||||
@@ -1064,96 +1049,82 @@ class DatabaseService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
row = cursor.fetchone()
|
row = cursor.fetchone()
|
||||||
conn.close()
|
|
||||||
|
|
||||||
if row:
|
if row:
|
||||||
return dict(row)
|
return dict(row)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _get_transaction_count(self, account_id=None, **filters):
|
def _get_transaction_count(self, account_id=None, **filters):
|
||||||
"""Get total count of transactions matching filters"""
|
"""Get total count of transactions matching filters"""
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return 0
|
return 0
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
query = "SELECT COUNT(*) FROM transactions WHERE 1=1"
|
with self._get_db_connection() as conn:
|
||||||
params = []
|
cursor = conn.cursor()
|
||||||
|
|
||||||
if account_id:
|
query = "SELECT COUNT(*) FROM transactions WHERE 1=1"
|
||||||
query += " AND accountId = ?"
|
params = []
|
||||||
params.append(account_id)
|
|
||||||
|
|
||||||
# Add same filters as get_transactions
|
if account_id:
|
||||||
if filters.get("date_from"):
|
query += " AND accountId = ?"
|
||||||
query += " AND transactionDate >= ?"
|
params.append(account_id)
|
||||||
params.append(filters["date_from"])
|
|
||||||
|
|
||||||
if filters.get("date_to"):
|
# Add same filters as get_transactions
|
||||||
query += " AND transactionDate <= ?"
|
if filters.get("date_from"):
|
||||||
params.append(filters["date_to"])
|
query += " AND transactionDate >= ?"
|
||||||
|
params.append(filters["date_from"])
|
||||||
|
|
||||||
if filters.get("min_amount") is not None:
|
if filters.get("date_to"):
|
||||||
query += " AND transactionValue >= ?"
|
query += " AND transactionDate <= ?"
|
||||||
params.append(filters["min_amount"])
|
params.append(filters["date_to"])
|
||||||
|
|
||||||
if filters.get("max_amount") is not None:
|
if filters.get("min_amount") is not None:
|
||||||
query += " AND transactionValue <= ?"
|
query += " AND transactionValue >= ?"
|
||||||
params.append(filters["max_amount"])
|
params.append(filters["min_amount"])
|
||||||
|
|
||||||
if filters.get("search"):
|
if filters.get("max_amount") is not None:
|
||||||
query += " AND description LIKE ?"
|
query += " AND transactionValue <= ?"
|
||||||
params.append(f"%{filters['search']}%")
|
params.append(filters["max_amount"])
|
||||||
|
|
||||||
|
if filters.get("search"):
|
||||||
|
query += " AND description LIKE ?"
|
||||||
|
params.append(f"%{filters['search']}%")
|
||||||
|
|
||||||
try:
|
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
count = cursor.fetchone()[0]
|
return cursor.fetchone()[0]
|
||||||
conn.close()
|
|
||||||
return count
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _persist_account(self, account_data: dict):
|
def _persist_account(self, account_data: dict):
|
||||||
"""Persist account details to SQLite database"""
|
"""Persist account details to SQLite database"""
|
||||||
db_path = path_manager.get_database_path()
|
with self._get_db_connection() as conn:
|
||||||
conn = sqlite3.connect(str(db_path))
|
cursor = conn.cursor()
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
# Create the accounts table if it doesn't exist
|
# Create the accounts table if it doesn't exist
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""CREATE TABLE IF NOT EXISTS accounts (
|
"""CREATE TABLE IF NOT EXISTS accounts (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
institution_id TEXT,
|
institution_id TEXT,
|
||||||
status TEXT,
|
status TEXT,
|
||||||
iban TEXT,
|
iban TEXT,
|
||||||
name TEXT,
|
name TEXT,
|
||||||
currency TEXT,
|
currency TEXT,
|
||||||
created DATETIME,
|
created DATETIME,
|
||||||
last_accessed DATETIME,
|
last_accessed DATETIME,
|
||||||
last_updated DATETIME,
|
last_updated DATETIME,
|
||||||
display_name TEXT,
|
display_name TEXT,
|
||||||
logo TEXT
|
logo TEXT
|
||||||
)"""
|
)"""
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create indexes for accounts table
|
# Create indexes for accounts table
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_accounts_institution_id
|
"""CREATE INDEX IF NOT EXISTS idx_accounts_institution_id
|
||||||
ON accounts(institution_id)"""
|
ON accounts(institution_id)"""
|
||||||
)
|
)
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""CREATE INDEX IF NOT EXISTS idx_accounts_status
|
"""CREATE INDEX IF NOT EXISTS idx_accounts_status
|
||||||
ON accounts(status)"""
|
ON accounts(status)"""
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
|
||||||
# First, check if account exists and preserve display_name
|
# First, check if account exists and preserve display_name
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"SELECT display_name FROM accounts WHERE id = ?", (account_data["id"],)
|
"SELECT display_name FROM accounts WHERE id = ?", (account_data["id"],)
|
||||||
@@ -1194,67 +1165,50 @@ class DatabaseService:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
|
||||||
|
|
||||||
return account_data
|
return account_data
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _get_accounts(self, account_ids=None):
|
def _get_accounts(self, account_ids=None):
|
||||||
"""Get account details from SQLite database"""
|
"""Get account details from SQLite database"""
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return []
|
return []
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
query = "SELECT * FROM accounts"
|
with self._get_db_connection(row_factory=True) as conn:
|
||||||
params = []
|
cursor = conn.cursor()
|
||||||
|
|
||||||
if account_ids:
|
query = "SELECT * FROM accounts"
|
||||||
placeholders = ",".join("?" * len(account_ids))
|
params = []
|
||||||
query += f" WHERE id IN ({placeholders})"
|
|
||||||
params.extend(account_ids)
|
|
||||||
|
|
||||||
query += " ORDER BY created DESC"
|
if account_ids:
|
||||||
|
placeholders = ",".join("?" * len(account_ids))
|
||||||
|
query += f" WHERE id IN ({placeholders})"
|
||||||
|
params.extend(account_ids)
|
||||||
|
|
||||||
|
query += " ORDER BY created DESC"
|
||||||
|
|
||||||
try:
|
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
|
|
||||||
accounts = [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
conn.close()
|
|
||||||
return accounts
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
def _get_account(self, account_id: str):
|
def _get_account(self, account_id: str):
|
||||||
"""Get specific account details from SQLite database"""
|
"""Get specific account details from SQLite database"""
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return None
|
return None
|
||||||
conn = sqlite3.connect(str(db_path))
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
try:
|
with self._get_db_connection(row_factory=True) as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
cursor.execute("SELECT * FROM accounts WHERE id = ?", (account_id,))
|
cursor.execute("SELECT * FROM accounts WHERE id = ?", (account_id,))
|
||||||
row = cursor.fetchone()
|
row = cursor.fetchone()
|
||||||
conn.close()
|
|
||||||
|
|
||||||
if row:
|
if row:
|
||||||
return dict(row)
|
return dict(row)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
except Exception as e:
|
@require_sqlite
|
||||||
conn.close()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
async def get_monthly_transaction_stats_from_db(
|
async def get_monthly_transaction_stats_from_db(
|
||||||
self,
|
self,
|
||||||
account_id: Optional[str] = None,
|
account_id: Optional[str] = None,
|
||||||
@@ -1262,10 +1216,6 @@ class DatabaseService:
|
|||||||
date_to: Optional[str] = None,
|
date_to: Optional[str] = None,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Get monthly transaction statistics aggregated by the database"""
|
"""Get monthly transaction statistics aggregated by the database"""
|
||||||
if not self.sqlite_enabled:
|
|
||||||
logger.warning("SQLite database disabled, cannot read monthly stats")
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
db_path = path_manager.get_database_path()
|
db_path = path_manager.get_database_path()
|
||||||
monthly_stats = self.analytics_processor.calculate_monthly_stats(
|
monthly_stats = self.analytics_processor.calculate_monthly_stats(
|
||||||
|
|||||||
Reference in New Issue
Block a user