feat: Change default database engine to SQLite, change schema.

leggen/commands/sync.py

@@ -1,55 +1,99 @@
+from datetime import datetime
+
 import click
-from pymongo import MongoClient
-from pymongo.errors import DuplicateKeyError
 
 from leggen.main import cli
+from leggen.utils.mongo import save_transactions as save_transactions_mongo
 from leggen.utils.network import get
-from leggen.utils.text import error, info, success, warning
+from leggen.utils.sqlite import save_transactions as save_transactions_sqlite
+from leggen.utils.text import error, info
 
 
 def save_transactions(ctx: click.Context, account: str):
+    info(f"[{account}] Getting account details")
+    account_info = get(ctx, f"/accounts/{account}")
+
     info(f"[{account}] Getting transactions")
-    all_transactions = []
+    transactions = []
+
     account_transactions = get(ctx, f"/accounts/{account}/transactions/").get(
         "transactions", []
     )
 
     for transaction in account_transactions.get("booked", []):
-        transaction["accountId"] = account
-        transaction["transactionStatus"] = "booked"
-        all_transactions.append(transaction)
+        booked_date = datetime.fromisoformat(
+            transaction.get("bookingDateTime", transaction.get("bookingDate"))
+        )
+        value_date = datetime.fromisoformat(
+            transaction.get("valueDateTime", transaction.get("valueDate"))
+        )
+        min_date = min(booked_date, value_date)
+
+        transactionValue = float(
+            transaction.get("transactionAmount", {}).get("amount", 0)
+        )
+        currency = transaction.get("transactionAmount", {}).get("currency", "")
+
+        description = transaction.get(
+            "remittanceInformationUnstructured",
+            ",".join(transaction.get("remittanceInformationUnstructuredArray", [])),
+        )
+
+        t = {
+            "internalTransactionId": transaction.get("internalTransactionId"),
+            "institutionId": account_info["institution_id"],
+            "iban": account_info.get("iban", "N/A"),
+            "transactionDate": min_date,
+            "description": description,
+            "transactionValue": transactionValue,
+            "transactionCurrency": currency,
+            "transactionStatus": "booked",
+            "accountId": account,
+            "rawTransaction": transaction,
+        }
+        transactions.append(t)
 
     for transaction in account_transactions.get("pending", []):
-        transaction["accountId"] = account
-        transaction["transactionStatus"] = "pending"
-        all_transactions.append(transaction)
+        booked_date = datetime.fromisoformat(
+            transaction.get("bookingDateTime", transaction.get("bookingDate"))
+        )
+        value_date = datetime.fromisoformat(
+            transaction.get("valueDateTime", transaction.get("valueDate"))
+        )
+        min_date = min(booked_date, value_date)
 
-    info(f"[{account}] Fetched {len(all_transactions)} transactions, saving to MongoDB")
+        transactionValue = float(
+            transaction.get("transactionAmount", {}).get("amount", 0)
+        )
+        currency = transaction.get("transactionAmount", {}).get("currency", "")
 
-    # Connect to MongoDB
-    mongo_uri = ctx.obj["mongo_uri"]
-    client = MongoClient(mongo_uri)
-    db = client["leggen"]
-    transactions_collection = db["transactions"]
+        description = transaction.get(
+            "remittanceInformationUnstructured",
+            ",".join(transaction.get("remittanceInformationUnstructuredArray", [])),
+        )
 
-    # Create a unique index on transactionId
-    transactions_collection.create_index("transactionId", unique=True)
+        t = {
+            "internalTransactionId": transaction.get("internalTransactionId"),
+            "institutionId": account_info["institution_id"],
+            "iban": account_info.get("iban", "N/A"),
+            "transactionDate": min_date,
+            "description": description,
+            "transactionValue": transactionValue,
+            "transactionCurrency": currency,
+            "transactionStatus": "pending",
+            "accountId": account,
+            "rawTransaction": transaction,
+        }
+        transactions.append(t)
 
-    # Insert transactions into MongoDB
-    new_transactions_count = 0
-    duplicates_count = 0
-
-    for transaction in all_transactions:
-        try:
-            transactions_collection.insert_one(transaction)
-            new_transactions_count += 1
-        except DuplicateKeyError:
-            # A transaction with the same ID already exists, skip insertion
-            duplicates_count += 1
-
-    success(f"[{account}] Inserted {new_transactions_count} new transactions")
-    if duplicates_count:
-        warning(f"[{account}] Skipped {duplicates_count} duplicate transactions")
+    sqlite = ctx.obj["sqlite"]
+    info(
+        f"[{account}] Fetched {len(transactions)} transactions, saving to {'SQLite' if sqlite else 'MongoDB'}"
+    )
+    if sqlite:
+        save_transactions_sqlite(ctx, account, transactions)
+    else:
+        save_transactions_mongo(ctx, account, transactions)
 
 
 @cli.command()