nikdoof/leggen
1
1
Fork 0
mirror of https://github.com/elisiariocouto/leggen.git synced 2025-12-13 19:32:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Remove MongoDB support, simplify to SQLite-only architecture

Browse Source 
- Remove pymongo dependency from pyproject.toml and update lock file
- Delete leggen/database/mongo.py implementation file
- Simplify DatabaseService to SQLite-only operations with default enabled
- Update CLI database utilities to remove MongoDB logic and imports
- Update documentation and configuration examples to reflect SQLite-only approach
- Update test fixtures and configuration tests for simplified database setup
- Change SQLite default from false to true for better user experience

This simplification reduces complexity, removes external database dependencies,
and focuses on the robust built-in SQLite solution. All 46 tests passing.

Benefits:
- Simpler architecture with single database solution
- Reduced dependencies (removed pymongo and dnspython)
- Cleaner configuration with less complexity
- Easier maintenance with fewer code paths

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Elisiário Couto
2025-09-02 00:46:12 +01:00
committed by Elisiário Couto
parent 34e793c75c
commit 47164e8546
9 changed files with 24 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.claude/settings.local.json
View File

@@ -3,7 +3,8 @@
"allow": [ "allow": [
"Bash(mkdir:*)", "Bash(mkdir:*)",
"Bash(uv sync:*)", "Bash(uv sync:*)",
"Bash(uv run pytest:*)" "Bash(uv run pytest:*)",
"Bash(git commit:*)"
], ],
"deny": [], "deny": [],
"ask": [] "ask": []

8
README.md
View File

@@ -17,7 +17,6 @@ Having your bank data accessible through both CLI and REST API gives you the pow
### 📦 Storage ### 📦 Storage
- [SQLite](https://www.sqlite.org): for storing transactions, simple and easy to use - [SQLite](https://www.sqlite.org): for storing transactions, simple and easy to use
- [MongoDB](https://www.mongodb.com/docs/): alternative store for transactions, good balance between performance and query capabilities
### 📊 Visualization ### 📊 Visualization
- [NocoDB](https://github.com/nocodb/nocodb): for visualizing and querying transactions, a simple and easy to use interface for SQLite - [NocoDB](https://github.com/nocodb/nocodb): for visualizing and querying transactions, a simple and easy to use interface for SQLite
@@ -32,7 +31,7 @@ Having your bank data accessible through both CLI and REST API gives you the pow
- Support for both booked and pending transactions - Support for both booked and pending transactions
### 🔄 Data Management ### 🔄 Data Management
- Sync all transactions with SQLite and/or MongoDB databases - Sync all transactions with SQLite database
- Background sync scheduling with configurable cron expressions - Background sync scheduling with configurable cron expressions
- Automatic transaction deduplication and status tracking - Automatic transaction deduplication and status tracking
- Real-time sync status monitoring - Real-time sync status monitoring
@@ -105,11 +104,6 @@ url = "https://bankaccountdata.gocardless.com/api/v2"
[database] [database]
sqlite = true sqlite = true
mongodb = false
# Optional: MongoDB configuration
[database.mongodb]
uri = "mongodb://localhost:27017"
# Optional: Background sync scheduling # Optional: Background sync scheduling
[scheduler.sync] [scheduler.sync]

54
leggen/database/mongo.py
View File

@@ -1,54 +0,0 @@
import click
from pymongo import MongoClient
from pymongo.errors import DuplicateKeyError
from leggen.utils.text import success, warning
def persist_balances(ctx: click.Context, balance: dict) -> None:
# Connect to MongoDB
mongo_uri = ctx.obj.get("database", {}).get("mongodb", {}).get("uri")
client = MongoClient(mongo_uri)
db = client["leggen"]
balances_collection = db["balances"]
# Insert balance into MongoDB
try:
balances_collection.insert_one(balance)
success(
f"[{balance['account_id']}] Inserted new balance if type {balance['type']}"
)
except DuplicateKeyError:
warning(f"[{balance['account_id']}] Skipped duplicate balance")
client.close()
def persist_transactions(ctx: click.Context, account: str, transactions: list) -> list:
# Connect to MongoDB
mongo_uri = ctx.obj.get("database", {}).get("mongodb", {}).get("uri")
client = MongoClient(mongo_uri)
db = client["leggen"]
transactions_collection = db["transactions"]
# Create a unique index on internalTransactionId
transactions_collection.create_index("internalTransactionId", unique=True)
# Insert transactions into MongoDB
duplicates_count = 0
new_transactions = []
for transaction in transactions:
try:
transactions_collection.insert_one(transaction)
new_transactions.append(transaction)
except DuplicateKeyError:
# A transaction with the same ID already exists, skip insertion
duplicates_count += 1
success(f"[{account}] Inserted {len(new_transactions)} new transactions")
if duplicates_count:
warning(f"[{account}] Skipped {duplicates_count} duplicate transactions")
return new_transactions

32
leggen/utils/database.py
View File

@@ -2,43 +2,33 @@ from datetime import datetime
import click import click
import leggen.database.mongo as mongodb_engine
import leggen.database.sqlite as sqlite_engine import leggen.database.sqlite as sqlite_engine
from leggen.utils.network import get from leggen.utils.network import get
from leggen.utils.text import info, warning from leggen.utils.text import info, warning
def persist_balance(ctx: click.Context, account: str, balance: dict) -> None: def persist_balance(ctx: click.Context, account: str, balance: dict) -> None:
sqlite = ctx.obj.get("database", {}).get("sqlite", False) sqlite = ctx.obj.get("database", {}).get("sqlite", True)
mongodb = ctx.obj.get("database", {}).get("mongodb", False)
if not sqlite and not mongodb: if not sqlite:
warning("No database engine is enabled, skipping balance saving") warning("SQLite database is disabled, skipping balance saving")
return
if sqlite: info(f"[{account}] Fetched balances, saving to SQLite")
info(f"[{account}] Fetched balances, saving to SQLite") sqlite_engine.persist_balances(ctx, balance)
sqlite_engine.persist_balances(ctx, balance)
else:
info(f"[{account}] Fetched balances, saving to MongoDB")
mongodb_engine.persist_balances(ctx, balance)
def persist_transactions(ctx: click.Context, account: str, transactions: list) -> list: def persist_transactions(ctx: click.Context, account: str, transactions: list) -> list:
sqlite = ctx.obj.get("database", {}).get("sqlite", False) sqlite = ctx.obj.get("database", {}).get("sqlite", True)
mongodb = ctx.obj.get("database", {}).get("mongodb", False)
if not sqlite and not mongodb: if not sqlite:
warning("No database engine is enabled, skipping transaction saving") warning("SQLite database is disabled, skipping transaction saving")
# WARNING: This will return the transactions list as is, without saving it to any database # WARNING: This will return the transactions list as is, without saving it to any database
# Possible duplicate notifications will be sent if the filters are enabled # Possible duplicate notifications will be sent if the filters are enabled
return transactions return transactions
if sqlite: info(f"[{account}] Fetched {len(transactions)} transactions, saving to SQLite")
info(f"[{account}] Fetched {len(transactions)} transactions, saving to SQLite") return sqlite_engine.persist_transactions(ctx, account, transactions)
return sqlite_engine.persist_transactions(ctx, account, transactions)
else:
info(f"[{account}] Fetched {len(transactions)} transactions, saving to MongoDB")
return mongodb_engine.persist_transactions(ctx, account, transactions)
def save_transactions(ctx: click.Context, account: str) -> list: def save_transactions(ctx: click.Context, account: str) -> list:

35
leggend/services/database_service.py
View File

@@ -9,33 +9,23 @@ from leggend.config import config
class DatabaseService: class DatabaseService:
def __init__(self): def __init__(self):
self.db_config = config.database_config self.db_config = config.database_config
self.sqlite_enabled = self.db_config.get("sqlite", False) self.sqlite_enabled = self.db_config.get("sqlite", True)
self.mongodb_enabled = self.db_config.get("mongodb", False)
async def persist_balance(self, account_id: str, balance_data: Dict[str, Any]) -> None: async def persist_balance(self, account_id: str, balance_data: Dict[str, Any]) -> None:
"""Persist account balance data""" """Persist account balance data"""
if not self.sqlite_enabled and not self.mongodb_enabled: if not self.sqlite_enabled:
logger.warning("No database engine enabled, skipping balance persistence") logger.warning("SQLite database disabled, skipping balance persistence")
return return
if self.sqlite_enabled: await self._persist_balance_sqlite(account_id, balance_data)
await self._persist_balance_sqlite(account_id, balance_data)
if self.mongodb_enabled:
await self._persist_balance_mongodb(account_id, balance_data)
async def persist_transactions(self, account_id: str, transactions: List[Dict[str, Any]]) -> List[Dict[str, Any]]: async def persist_transactions(self, account_id: str, transactions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Persist transactions and return new transactions""" """Persist transactions and return new transactions"""
if not self.sqlite_enabled and not self.mongodb_enabled: if not self.sqlite_enabled:
logger.warning("No database engine enabled, skipping transaction persistence") logger.warning("SQLite database disabled, skipping transaction persistence")
return transactions return transactions
if self.sqlite_enabled: return await self._persist_transactions_sqlite(account_id, transactions)
return await self._persist_transactions_sqlite(account_id, transactions)
elif self.mongodb_enabled:
return await self._persist_transactions_mongodb(account_id, transactions)
return []
def process_transactions(self, account_id: str, account_info: Dict[str, Any], transaction_data: Dict[str, Any]) -> List[Dict[str, Any]]: def process_transactions(self, account_id: str, account_info: Dict[str, Any], transaction_data: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Process raw transaction data into standardized format""" """Process raw transaction data into standardized format"""
@@ -96,19 +86,8 @@ class DatabaseService:
# Would import and use leggen.database.sqlite # Would import and use leggen.database.sqlite
logger.info(f"Persisting balance to SQLite for account {account_id}") logger.info(f"Persisting balance to SQLite for account {account_id}")
async def _persist_balance_mongodb(self, account_id: str, balance_data: Dict[str, Any]) -> None:
"""Persist balance to MongoDB - placeholder implementation"""
# Would import and use leggen.database.mongo
logger.info(f"Persisting balance to MongoDB for account {account_id}")
async def _persist_transactions_sqlite(self, account_id: str, transactions: List[Dict[str, Any]]) -> List[Dict[str, Any]]: async def _persist_transactions_sqlite(self, account_id: str, transactions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Persist transactions to SQLite - placeholder implementation""" """Persist transactions to SQLite - placeholder implementation"""
# Would import and use leggen.database.sqlite # Would import and use leggen.database.sqlite
logger.info(f"Persisting {len(transactions)} transactions to SQLite for account {account_id}") logger.info(f"Persisting {len(transactions)} transactions to SQLite for account {account_id}")
return transactions # Return new transactions for notifications return transactions # Return new transactions for notifications
async def _persist_transactions_mongodb(self, account_id: str, transactions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Persist transactions to MongoDB - placeholder implementation"""
# Would import and use leggen.database.mongo
logger.info(f"Persisting {len(transactions)} transactions to MongoDB for account {account_id}")
return transactions # Return new transactions for notifications

2
pyproject.toml
View File

@@ -11,7 +11,6 @@ keywords = [
"cli", "cli",
"psd2", "psd2",
"gocardless", "gocardless",
"mongodb",
"bank", "bank",
"transactions", "transactions",
"finance", "finance",
@@ -29,7 +28,6 @@ dependencies = [
"requests>=2.31.0,<3", "requests>=2.31.0,<3",
"loguru>=0.7.2,<0.8", "loguru>=0.7.2,<0.8",
"tabulate>=0.9.0,<0.10", "tabulate>=0.9.0,<0.10",
"pymongo>=4.6.1,<5",
"discord-webhook>=1.3.1,<2", "discord-webhook>=1.3.1,<2",
"fastapi>=0.104.0,<1", "fastapi>=0.104.0,<1",
"uvicorn[standard]>=0.24.0,<1", "uvicorn[standard]>=0.24.0,<1",

3
tests/conftest.py
View File

@@ -29,8 +29,7 @@ def mock_config(temp_config_dir):
"url": "https://bankaccountdata.gocardless.com/api/v2" "url": "https://bankaccountdata.gocardless.com/api/v2"
}, },
"database": { "database": {
"sqlite": True, "sqlite": True
"mongodb": False
}, },
"scheduler": { "scheduler": {
"sync": { "sync": {

4
tests/unit/test_config.py
View File

@@ -107,7 +107,7 @@ class TestConfig:
def test_update_section_success(self, temp_config_dir): def test_update_section_success(self, temp_config_dir):
"""Test updating entire configuration section.""" """Test updating entire configuration section."""
initial_config = { initial_config = {
"database": {"sqlite": True, "mongodb": False} "database": {"sqlite": True}
} }
config_file = temp_config_dir / "config.toml" config_file = temp_config_dir / "config.toml"
@@ -119,7 +119,7 @@ class TestConfig:
config._config = None config._config = None
config.load_config(str(config_file)) config.load_config(str(config_file))
new_db_config = {"sqlite": False, "mongodb": True, "uri": "mongodb://localhost"} new_db_config = {"sqlite": False, "path": "./custom.db"}
config.update_section("database", new_db_config) config.update_section("database", new_db_config)
assert config.database_config == new_db_config assert config.database_config == new_db_config

31
uv.lock generated
View File

@@ -119,15 +119,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973, upload-time = "2024-10-09T18:35:44.272Z" }, { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973, upload-time = "2024-10-09T18:35:44.272Z" },
] ]
[[package]]
name = "dnspython"
version = "2.7.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/b5/4a/263763cb2ba3816dd94b08ad3a33d5fdae34ecb856678773cc40a3605829/dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1", size = 345197, upload-time = "2024-10-05T20:14:59.362Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload-time = "2024-10-05T20:14:57.687Z" },
]
[[package]] [[package]]
name = "fastapi" name = "fastapi"
version = "0.116.1" version = "0.116.1"
@@ -241,7 +232,6 @@ dependencies = [
{ name = "fastapi" }, { name = "fastapi" },
{ name = "httpx" }, { name = "httpx" },
{ name = "loguru" }, { name = "loguru" },
{ name = "pymongo" },
{ name = "requests" }, { name = "requests" },
{ name = "tabulate" }, { name = "tabulate" },
{ name = "tomli-w" }, { name = "tomli-w" },
@@ -267,7 +257,6 @@ requires-dist = [
{ name = "fastapi", specifier = ">=0.104.0,<1" }, { name = "fastapi", specifier = ">=0.104.0,<1" },
{ name = "httpx", specifier = ">=0.28.1" }, { name = "httpx", specifier = ">=0.28.1" },
{ name = "loguru", specifier = ">=0.7.2,<0.8" }, { name = "loguru", specifier = ">=0.7.2,<0.8" },
{ name = "pymongo", specifier = ">=4.6.1,<5" },
{ name = "requests", specifier = ">=2.31.0,<3" }, { name = "requests", specifier = ">=2.31.0,<3" },
{ name = "tabulate", specifier = ">=0.9.0,<0.10" }, { name = "tabulate", specifier = ">=0.9.0,<0.10" },
{ name = "tomli-w", specifier = ">=1.0.0,<2" }, { name = "tomli-w", specifier = ">=1.0.0,<2" },
@@ -399,26 +388,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
] ]
[[package]]
name = "pymongo"
version = "4.10.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "dnspython" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1a/35/b62a3139f908c68b69aac6a6a3f8cc146869de0a7929b994600e2c587c77/pymongo-4.10.1.tar.gz", hash = "sha256:a9de02be53b6bb98efe0b9eda84ffa1ec027fcb23a2de62c4f941d9a2f2f3330", size = 1903902, upload-time = "2024-10-01T23:07:58.525Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/10/d1/60ad99fe3f64d45e6c71ac0e3078e88d9b64112b1bae571fc3707344d6d1/pymongo-4.10.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fbedc4617faa0edf423621bb0b3b8707836687161210d470e69a4184be9ca011", size = 943356, upload-time = "2024-10-01T23:06:50.9Z" },
{ url = "https://files.pythonhosted.org/packages/ca/9b/21d4c6b4ee9c1fa9691c68dc2a52565e0acb644b9e95148569b4736a4ebd/pymongo-4.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7bd26b2aec8ceeb95a5d948d5cc0f62b0eb6d66f3f4230705c1e3d3d2c04ec76", size = 943142, upload-time = "2024-10-01T23:06:52.146Z" },
{ url = "https://files.pythonhosted.org/packages/07/af/691b7454e219a8eb2d1641aecedd607e3a94b93650c2011ad8a8fd74ef9f/pymongo-4.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb104c3c2a78d9d85571c8ac90ec4f95bca9b297c6eee5ada71fabf1129e1674", size = 1909129, upload-time = "2024-10-01T23:06:53.551Z" },
{ url = "https://files.pythonhosted.org/packages/0c/74/fd75d5ad4181d6e71ce0fca32404fb71b5046ac84d9a1a2f0862262dd032/pymongo-4.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4924355245a9c79f77b5cda2db36e0f75ece5faf9f84d16014c0a297f6d66786", size = 1987763, upload-time = "2024-10-01T23:06:55.304Z" },
{ url = "https://files.pythonhosted.org/packages/8a/56/6d3d0ef63c6d8cb98c7c653a3a2e617675f77a95f3853851d17a7664876a/pymongo-4.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:11280809e5dacaef4971113f0b4ff4696ee94cfdb720019ff4fa4f9635138252", size = 1950821, upload-time = "2024-10-01T23:06:57.541Z" },
{ url = "https://files.pythonhosted.org/packages/70/ed/1603fa0c0e51444752c3fa91f16c3a97e6d92eb9fe5e553dae4f18df16f6/pymongo-4.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5d55f2a82e5eb23795f724991cac2bffbb1c0f219c0ba3bf73a835f97f1bb2e", size = 1912247, upload-time = "2024-10-01T23:06:59.023Z" },
{ url = "https://files.pythonhosted.org/packages/c1/66/e98b2308971d45667cb8179d4d66deca47336c90663a7e0527589f1038b7/pymongo-4.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e974ab16a60be71a8dfad4e5afccf8dd05d41c758060f5d5bda9a758605d9a5d", size = 1862230, upload-time = "2024-10-01T23:07:01.407Z" },
{ url = "https://files.pythonhosted.org/packages/6c/80/ba9b7ed212a5f8cf8ad7037ed5bbebc1c587fc09242108f153776e4a338b/pymongo-4.10.1-cp312-cp312-win32.whl", hash = "sha256:544890085d9641f271d4f7a47684450ed4a7344d6b72d5968bfae32203b1bb7c", size = 903045, upload-time = "2024-10-01T23:07:02.973Z" },
{ url = "https://files.pythonhosted.org/packages/76/8b/5afce891d78159912c43726fab32641e3f9718f14be40f978c148ea8db48/pymongo-4.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:dcc07b1277e8b4bf4d7382ca133850e323b7ab048b8353af496d050671c7ac52", size = 926686, upload-time = "2024-10-01T23:07:04.403Z" },
]
[[package]] [[package]]
name = "pytest" name = "pytest"
version = "8.4.1" version = "8.4.1"