chore(ci): Bump version to 0.2.0

.gitignore

@@ -160,3 +160,5 @@ cython_debug/
 #.idea/
 data/
 docker-compose.dev.yml
+nocodb/
+sql/

CHANGELOG.md

@@ -1,3 +1,21 @@
+## 0.2.0 (2024/02/27)
+
+### Bug Fixes
+
+- **compose:** Fix ofelia configuration, add sync command as the default. ([433d1737](https://github.com/elisiariocouto/leggen/commit/433d17371ead323ca9b793a2dd5782cca598ffcf))
+
+
+### Documentation
+
+-  Improve README.md. ([de17cf44](https://github.com/elisiariocouto/leggen/commit/de17cf44ec5260305de8aa053582744ec69d705f))
+
+
+### Features
+
+-  Add periodic sync, handled by ofelia. ([91c74b04](https://github.com/elisiariocouto/leggen/commit/91c74b0412713ef8305fbe7fcf7c53e4cf8948fe))
+-  Change default database engine to SQLite, change schema. ([f9ab3ae0](https://github.com/elisiariocouto/leggen/commit/f9ab3ae0a813f2a512b4f5fa57e0da089f823783))
+
+
 ## 0.1.1 (2024/02/18)
 
 ### Bug Fixes

README.md

@@ -1,22 +1,35 @@
-# leggen
+# 💲 leggen
 
 An Open Banking CLI.
 
-## Features
+This tool aims to provide a simple way to connect to banks using the GoCardless Open Banking API.
+
+Having a simple CLI tool to connect to banks and list transactions can be very useful for developers and companies that need to access bank data.
+
+Having your bank data in a database, gives you the power to backup, analyze and create reports with your data.
+
+## 🛠️ Technologies
+  - Python: for the CLI
+  - [GoCardless Open Banking API](https://developer.gocardless.com/bank-account-data/overview): for connecting to banks
+  - [SQLite](https://www.sqlite.org): for storing transactions, simple and easy to use
+  - [MongoDB](https://www.mongodb.com/docs/): alternative store for transactions, good balance between performance and query capabilities
+  - [Ofelia](https://github.com/mcuadros/ofelia): for scheduling regular syncs with the database when using Docker
+
+## ✨ Features
   - Connect to banks using GoCardless Open Banking API
   - List all connected banks and their status
   - List balances of all connected accounts
-  - List transactions for an account
+  - List transactions for all connected accounts
   - Sync all transactions with a MongoDB database
 
-## Installation and Configuration
+## 🚀 Installation and Configuration
 
 In order to use `leggen`, you need to create a GoCardless account. GoCardless is a service that provides access to Open Banking APIs. You can create an account at https://gocardless.com/bank-account-data/.
 
 After creating an account and getting your API keys, the best way is to use the [compose file](docker-compose.yml). Open the file and adapt it to your needs. Then run the following command:
 
 ```bash
-$ docker-compose up -d
+$ docker compose up -d
 ```
 
 The leggen container will exit, this is expected. Now you can run the following command to create the configuration file:
@@ -37,7 +50,7 @@ To sync all transactions with the database, run the following command:
 $ docker compose run leggen sync
 ```
 
-## Usage
+## 👩‍🏫 Usage
 
 ```
 $ leggen --help
@@ -60,5 +73,5 @@ Commands:
   transactions  List transactions for an account
 ```
 
-## Caveats
-  - This project is still in early development.
+## ⚠️ Caveats
+  - This project is still in early development, breaking changes may occur.

docker-compose.yml

@@ -1,31 +1,57 @@
 version: '3.1'
 
 services:
-  mongo:
-    image: mongo:7
-    restart: "unless-stopped"
-    # If you want to expose the mongodb port to the host, uncomment the following lines
-    # ports:
-    #   - 127.0.0.1:27017:27017
-    volumes:
-      - "./data:/data/db"
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: "leggen"
-      MONGO_INITDB_ROOT_PASSWORD: "changeme"
-
+  # Defaults to `sync` command.
   leggen:
     image: elisiariocouto/leggen:latest
+    command: sync
     restart: "no"
     environment:
-      LEGGEN_MONGO_URI: mongodb://leggen:changeme@mongo:27017/
       LEGGEN_GC_API_KEY: "changeme"
       LEGGEN_GC_API_SECRET: "changeme"
+      # Uncomment the following lines if you use MongoDB
+      # LEGGEN_MONGO_URI: "mongodb://leggen:changeme@mongo:27017/leggen"
     volumes:
-      - "./leggen:/root/.config/leggen"
+      - "./leggen:/root"
     depends_on:
       - mongo
 
-  # If you want to have an admin interface for your mongodb, uncomment the following lines
+  nocodb:
+    image: nocodb/nocodb:latest
+    restart: "unless-stopped"
+    volumes:
+      - "./nocodb:/usr/app/data/"
+      - "./leggen:/usr/leggen:ro"
+    ports:
+      - "127.0.0.1:8080:8080"
+
+  # Recommended: Run `leggen sync` every day.
+  ofelia:
+    image: mcuadros/ofelia:latest
+    restart: "unless-stopped"
+    depends_on:
+      - leggen
+    command: daemon --docker -f label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    labels:
+      ofelia.job-run.leggen-sync.schedule: "0 0 3 * * *"
+      ofelia.job-run.leggen-sync.container: ${COMPOSE_PROJECT_NAME}-leggen-1
+
+  # Optional: If you want to have a mongodb, uncomment the following lines
+  # mongo:
+  #   image: mongo:7
+  #   restart: "unless-stopped"
+  #   # If you want to expose the mongodb port to the host, uncomment the following lines
+  #   # ports:
+  #   #   - 127.0.0.1:27017:27017
+  #   volumes:
+  #     - "./data:/data/db"
+  #   environment:
+  #     MONGO_INITDB_ROOT_USERNAME: "leggen"
+  #     MONGO_INITDB_ROOT_PASSWORD: "changeme"
+
+  # Optional: If you want to have an admin interface for your mongodb, uncomment the following lines
   # mongo-express:
   #   image: mongo-express
   #   restart: "unless-stopped"

leggen/commands/init.py

@@ -7,7 +7,11 @@ from leggen.utils.config import save_config
 
 @cli.command()
 @click.option(
-    "--api-key", prompt=True, help="GoCardless API Key", envvar="LEGGEN_GC_API_KEY"
+    "--api-key",
+    prompt=True,
+    help="GoCardless API Key",
+    envvar="LEGGEN_GC_API_KEY",
+    show_envvar=True,
 )
 @click.option(
     "--api-secret",
@@ -15,6 +19,7 @@ from leggen.utils.config import save_config
     help="GoCardless API Secret",
     hide_input=True,
     envvar="LEGGEN_GC_API_SECRET",
+    show_envvar=True,
 )
 @click.option(
     "--api-url",
@@ -22,10 +27,32 @@ from leggen.utils.config import save_config
     help="GoCardless API URL",
     show_default=True,
     envvar="LEGGEN_GC_API_URL",
+    show_envvar=True,
+)
+@click.option(
+    "--sqlite/--mongo",
+    prompt=True,
+    default=True,
+    help="Use SQLite or MongoDB",
+    show_default=True,
+)
+@click.option(
+    "--mongo-uri",
+    prompt=True,
+    help="MongoDB URI",
+    envvar="LEGGEN_MONGO_URI",
+    show_envvar=True,
+    default="mongodb://localhost:27017",
 )
-@click.option("--mongo-uri", prompt=True, help="MongoDB URI", envvar="LEGGEN_MONGO_URI")
 @click.pass_context
-def init(ctx: click.Context, api_key, api_secret, api_url, mongo_uri):
+def init(
+    ctx: click.Context,
+    api_key: str,
+    api_secret: str,
+    api_url: str,
+    sqlite: bool,
+    mongo_uri: str,
+):
     """
     Create configuration file
     """
@@ -33,6 +60,7 @@ def init(ctx: click.Context, api_key, api_secret, api_url, mongo_uri):
         "api_key": api_key,
         "api_secret": api_secret,
         "api_url": api_url,
+        "sqlite": sqlite,
         "mongo_uri": mongo_uri,
     }
 

leggen/commands/sync.py

@@ -1,55 +1,99 @@
+from datetime import datetime
+
 import click
-from pymongo import MongoClient
-from pymongo.errors import DuplicateKeyError
 
 from leggen.main import cli
+from leggen.utils.mongo import save_transactions as save_transactions_mongo
 from leggen.utils.network import get
-from leggen.utils.text import error, info, success, warning
+from leggen.utils.sqlite import save_transactions as save_transactions_sqlite
+from leggen.utils.text import error, info
 
 
 def save_transactions(ctx: click.Context, account: str):
+    info(f"[{account}] Getting account details")
+    account_info = get(ctx, f"/accounts/{account}")
+
     info(f"[{account}] Getting transactions")
-    all_transactions = []
+    transactions = []
+
     account_transactions = get(ctx, f"/accounts/{account}/transactions/").get(
         "transactions", []
     )
 
     for transaction in account_transactions.get("booked", []):
-        transaction["accountId"] = account
-        transaction["transactionStatus"] = "booked"
-        all_transactions.append(transaction)
+        booked_date = datetime.fromisoformat(
+            transaction.get("bookingDateTime", transaction.get("bookingDate"))
+        )
+        value_date = datetime.fromisoformat(
+            transaction.get("valueDateTime", transaction.get("valueDate"))
+        )
+        min_date = min(booked_date, value_date)
+
+        transactionValue = float(
+            transaction.get("transactionAmount", {}).get("amount", 0)
+        )
+        currency = transaction.get("transactionAmount", {}).get("currency", "")
+
+        description = transaction.get(
+            "remittanceInformationUnstructured",
+            ",".join(transaction.get("remittanceInformationUnstructuredArray", [])),
+        )
+
+        t = {
+            "internalTransactionId": transaction.get("internalTransactionId"),
+            "institutionId": account_info["institution_id"],
+            "iban": account_info.get("iban", "N/A"),
+            "transactionDate": min_date,
+            "description": description,
+            "transactionValue": transactionValue,
+            "transactionCurrency": currency,
+            "transactionStatus": "booked",
+            "accountId": account,
+            "rawTransaction": transaction,
+        }
+        transactions.append(t)
 
     for transaction in account_transactions.get("pending", []):
-        transaction["accountId"] = account
-        transaction["transactionStatus"] = "pending"
-        all_transactions.append(transaction)
+        booked_date = datetime.fromisoformat(
+            transaction.get("bookingDateTime", transaction.get("bookingDate"))
+        )
+        value_date = datetime.fromisoformat(
+            transaction.get("valueDateTime", transaction.get("valueDate"))
+        )
+        min_date = min(booked_date, value_date)
 
-    info(f"[{account}] Fetched {len(all_transactions)} transactions, saving to MongoDB")
+        transactionValue = float(
+            transaction.get("transactionAmount", {}).get("amount", 0)
+        )
+        currency = transaction.get("transactionAmount", {}).get("currency", "")
 
-    # Connect to MongoDB
-    mongo_uri = ctx.obj["mongo_uri"]
-    client = MongoClient(mongo_uri)
-    db = client["leggen"]
-    transactions_collection = db["transactions"]
+        description = transaction.get(
+            "remittanceInformationUnstructured",
+            ",".join(transaction.get("remittanceInformationUnstructuredArray", [])),
+        )
 
-    # Create a unique index on transactionId
-    transactions_collection.create_index("transactionId", unique=True)
+        t = {
+            "internalTransactionId": transaction.get("internalTransactionId"),
+            "institutionId": account_info["institution_id"],
+            "iban": account_info.get("iban", "N/A"),
+            "transactionDate": min_date,
+            "description": description,
+            "transactionValue": transactionValue,
+            "transactionCurrency": currency,
+            "transactionStatus": "pending",
+            "accountId": account,
+            "rawTransaction": transaction,
+        }
+        transactions.append(t)
 
-    # Insert transactions into MongoDB
-    new_transactions_count = 0
-    duplicates_count = 0
-
-    for transaction in all_transactions:
-        try:
-            transactions_collection.insert_one(transaction)
-            new_transactions_count += 1
-        except DuplicateKeyError:
-            # A transaction with the same ID already exists, skip insertion
-            duplicates_count += 1
-
-    success(f"[{account}] Inserted {new_transactions_count} new transactions")
-    if duplicates_count:
-        warning(f"[{account}] Skipped {duplicates_count} duplicate transactions")
+    sqlite = ctx.obj["sqlite"]
+    info(
+        f"[{account}] Fetched {len(transactions)} transactions, saving to {'SQLite' if sqlite else 'MongoDB'}"
+    )
+    if sqlite:
+        save_transactions_sqlite(ctx, account, transactions)
+    else:
+        save_transactions_mongo(ctx, account, transactions)
 
 
 @cli.command()

leggen/commands/transactions.py

@@ -2,23 +2,15 @@ import click
 
 from leggen.main import cli
 from leggen.utils.network import get
-from leggen.utils.text import print_table
+from leggen.utils.text import info, print_table
 
 
-@cli.command()
-@click.argument("account", type=str)
-@click.pass_context
-def transactions(ctx: click.Context, account: str):
-    """
-    List transactions for an account
-
-    ACCOUNT is the account id, see 'leggen status' for the account ids
-    """
+def print_transactions(
+    ctx: click.Context, account_info: dict, account_transactions: dict
+):
+    info(f"Bank: {account_info['institution_id']}")
+    info(f"IBAN: {account_info.get('iban', 'N/A')}")
     all_transactions = []
-    account_transactions = get(ctx, f"/accounts/{account}/transactions/").get(
-        "transactions", []
-    )
-
     for transaction in account_transactions.get("booked", []):
         transaction["TYPE"] = "booked"
         all_transactions.append(transaction)
@@ -28,3 +20,33 @@ def transactions(ctx: click.Context, account: str):
         all_transactions.append(transaction)
 
     print_table(all_transactions)
+
+
+@cli.command()
+@click.option("-a", "--account", type=str, help="Account ID")
+@click.pass_context
+def transactions(ctx: click.Context, account: str):
+    """
+    List transactions
+
+    By default, this command lists all transactions for all accounts.
+
+    If the --account option is used, it will only list transactions for that account.
+    """
+    if account:
+        account_info = get(ctx, f"/accounts/{account}")
+        account_transactions = get(ctx, f"/accounts/{account}/transactions/").get(
+            "transactions", []
+        )
+        print_transactions(ctx, account_info, account_transactions)
+    else:
+        res = get(ctx, "/requisitions/")
+        accounts = []
+        for r in res["results"]:
+            accounts += r.get("accounts", [])
+        for account in accounts:
+            account_details = get(ctx, f"/accounts/{account}")
+            account_transactions = get(ctx, f"/accounts/{account}/transactions/").get(
+                "transactions", []
+            )
+            print_transactions(ctx, account_details, account_transactions)

leggen/main.py

@@ -98,5 +98,6 @@ def cli(ctx: click.Context):
     config = load_config()
     token = get_token(config)
     ctx.obj["api_url"] = config["api_url"]
+    ctx.obj["sqlite"] = config["sqlite"]
     ctx.obj["mongo_uri"] = config["mongo_uri"]
     ctx.obj["headers"] = {"Authorization": f"Bearer {token}"}

leggen/utils/mongo.py

@@ -0,0 +1,32 @@
+import click
+from pymongo import MongoClient
+from pymongo.errors import DuplicateKeyError
+
+from leggen.utils.text import success, warning
+
+
+def save_transactions(ctx: click.Context, account: str, transactions: list):
+    # Connect to MongoDB
+    mongo_uri = ctx.obj["mongo_uri"]
+    client = MongoClient(mongo_uri)
+    db = client["leggen"]
+    transactions_collection = db["transactions"]
+
+    # Create a unique index on internalTransactionId
+    transactions_collection.create_index("internalTransactionId", unique=True)
+
+    # Insert transactions into MongoDB
+    new_transactions_count = 0
+    duplicates_count = 0
+
+    for transaction in transactions:
+        try:
+            transactions_collection.insert_one(transaction)
+            new_transactions_count += 1
+        except DuplicateKeyError:
+            # A transaction with the same ID already exists, skip insertion
+            duplicates_count += 1
+
+    success(f"[{account}] Inserted {new_transactions_count} new transactions")
+    if duplicates_count:
+        warning(f"[{account}] Skipped {duplicates_count} duplicate transactions")

leggen/utils/sqlite.py

@@ -0,0 +1,80 @@
+import json
+import sqlite3
+from sqlite3 import IntegrityError
+
+import click
+
+from leggen.utils.text import success, warning
+
+
+def save_transactions(ctx: click.Context, account: str, transactions: list):
+    # Path to your SQLite database file
+
+    # Connect to SQLite database
+    conn = sqlite3.connect("./leggen.db")
+    cursor = conn.cursor()
+
+    # Create the transactions table if it doesn't exist
+    cursor.execute(
+        """CREATE TABLE IF NOT EXISTS transactions (
+        internalTransactionId TEXT PRIMARY KEY,
+        institutionId TEXT,
+        iban TEXT,
+        transactionDate DATETIME,
+        description TEXT,
+        transactionValue REAL,
+        transactionCurrency TEXT,
+        transactionStatus TEXT,
+        accountId TEXT,
+        rawTransaction JSON
+    )"""
+    )
+
+    # Insert transactions into SQLite database
+    new_transactions_count = 0
+    duplicates_count = 0
+
+    # Prepare an SQL statement for inserting data
+    insert_sql = """INSERT INTO transactions (
+        internalTransactionId,
+        institutionId,
+        iban,
+        transactionDate,
+        description,
+        transactionValue,
+        transactionCurrency,
+        transactionStatus,
+        accountId,
+        rawTransaction
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+
+    for transaction in transactions:
+        try:
+            cursor.execute(
+                insert_sql,
+                (
+                    transaction["internalTransactionId"],
+                    transaction["institutionId"],
+                    transaction["iban"],
+                    transaction["transactionDate"],
+                    transaction["description"],
+                    transaction["transactionValue"],
+                    transaction["transactionCurrency"],
+                    transaction["transactionStatus"],
+                    transaction["accountId"],
+                    json.dumps(transaction["rawTransaction"]),
+                ),
+            )
+
+            new_transactions_count += 1
+        except IntegrityError:
+            # A transaction with the same ID already exists, indicating a duplicate
+            duplicates_count += 1
+
+    # Commit changes and close the connection
+    conn.commit()
+    conn.close()
+
+    success(f"[{account}] Inserted {new_transactions_count} new transactions")
+    if duplicates_count:
+        warning(f"[{account}] Skipped {duplicates_count} duplicate transactions")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "leggen"
-version = "0.1.1"
+version = "0.2.0"
 description = "An Open Banking CLI"
 authors = ["Elisiário Couto <elisiario@couto.io>"]
 readme = "README.md"