From fcf08a4d898160426442bd80830f4431988f4313 Mon Sep 17 00:00:00 2001 From: Elias Schneider Date: Tue, 26 Nov 2024 20:14:31 +0100 Subject: [PATCH] feat!: add option to specify the Max Mind license key for the Geolite2 db --- .env.example | 3 +- .../workflows/build-and-push-docker-image.yml | 3 - .github/workflows/e2e-tests.yml | 3 - Dockerfile | 1 - README.md | 32 ++-- .../internal/bootstrap/router_bootstrap.go | 3 +- backend/internal/common/env_config.go | 4 + backend/internal/service/audit_log_service.go | 35 +---- backend/internal/service/geolite_service.go | 142 ++++++++++++++++++ scripts/download-ip-database.sh | 31 ---- 10 files changed, 168 insertions(+), 89 deletions(-) create mode 100644 backend/internal/service/geolite_service.go delete mode 100644 scripts/download-ip-database.sh diff --git a/.env.example b/.env.example index fae5478..ea26fd2 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ PUBLIC_APP_URL=http://localhost -TRUST_PROXY=false \ No newline at end of file +TRUST_PROXY=false +MAXMIND_LICENSE_KEY= \ No newline at end of file diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index d9c4cf9..be1bf6f 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -40,9 +40,6 @@ jobs: registry: ghcr.io username: ${{github.repository_owner}} password: ${{secrets.GITHUB_TOKEN}} - - - name: Download GeoLite2 City database - run: MAXMIND_LICENSE_KEY=${{ secrets.MAXMIND_LICENSE_KEY }} sh scripts/download-ip-database.sh - name: Build and push uses: docker/build-push-action@v4 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 41a3fae..5aff7f4 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -16,9 +16,6 @@ jobs: cache: 'npm' cache-dependency-path: frontend/package-lock.json - - name: Create dummy GeoLite2 City database - run: touch ./backend/GeoLite2-City.mmdb - - name: Build Docker Image run: docker build -t stonith404/pocket-id . diff --git a/Dockerfile b/Dockerfile index 8232a09..6346055 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,6 @@ COPY --from=frontend-builder /app/frontend/package.json ./frontend/package.json COPY --from=backend-builder /app/backend/pocket-id-backend ./backend/pocket-id-backend COPY --from=backend-builder /app/backend/migrations ./backend/migrations -COPY --from=backend-builder /app/backend/GeoLite2-City.mmdb ./backend/GeoLite2-City.mmdb COPY --from=backend-builder /app/backend/email-templates ./backend/email-templates COPY --from=backend-builder /app/backend/images ./backend/images diff --git a/README.md b/README.md index 935500e..a1283af 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,6 @@ Required tools: cd .. pm2 start pocket-id-backend --name pocket-id-backend - # Optional: Download the GeoLite2 city database. - # If not downloaded the ip location in the audit log will be empty. - MAXMIND_LICENSE_KEY= sh scripts/download-ip-database.sh - # Start the frontend cd ../frontend npm install @@ -130,9 +126,6 @@ docker compose up -d cd .. pm2 start pocket-id-backend --name pocket-id-backend - # Optional: Update the GeoLite2 city database - MAXMIND_LICENSE_KEY= sh scripts/download-ip-database.sh - # Start the frontend cd ../frontend npm install @@ -146,17 +139,20 @@ docker compose up -d ## Environment variables -| Variable | Default Value | Recommended to change | Description | -| ---------------------- | ----------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PUBLIC_APP_URL` | `http://localhost` | yes | The URL where you will access the app. | -| `TRUST_PROXY` | `false` | yes | Whether the app is behind a reverse proxy. | -| `PUID` and `PGID` | `1000` | yes | The user and group ID of the user who should run Pocket ID inside the Docker container and owns the files that are mounted with the volume. You can get the `PUID` and `GUID` of your user on your host machine by using the command `id`. For more information see [this article](https://docs.linuxserver.io/general/understanding-puid-and-pgid/#using-the-variables). | -| `DB_PATH` | `data/pocket-id.db` | no | The path to the SQLite database. | -| `UPLOAD_PATH` | `data/uploads` | no | The path where the uploaded files are stored. | -| `INTERNAL_BACKEND_URL` | `http://localhost:8080` | no | The URL where the backend is accessible. | -| `CADDY_PORT` | `80` | no | The port on which Caddy should listen. Caddy is only active inside the Docker container. If you want to change the exposed port of the container then you sould change this variable. | -| `PORT` | `3000` | no | The port on which the frontend should listen. | -| `BACKEND_PORT` | `8080` | no | The port on which the backend should listen. | +| Variable | Default Value | Recommended to change | Description | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PUBLIC_APP_URL` | `http://localhost` | yes | The URL where you will access the app. | +| `TRUST_PROXY` | `false` | yes | Whether the app is behind a reverse proxy. | +| `MAXMIND_LICENSE_KEY` | `-` | yes | # License key for MaxMind GeoIP2lite. | +| License key for the GeoLite2 database. This is necessary to get the location of the ip adresses in the audit log. If not specified, the location will be unknown. You can get the license key here. | +| `PUID` and `PGID` | `1000` | yes | The user and group ID of the user who should run Pocket ID inside the Docker container and owns the files that are mounted with the volume. You can get the `PUID` and `GUID` of your user on your host machine by using the command `id`. For more information see [this article](https://docs.linuxserver.io/general/understanding-puid-and-pgid/#using-the-variables). | +| `DB_PATH` | `data/pocket-id.db` | no | The path to the SQLite database. | +| `UPLOAD_PATH` | `data/uploads` | no | The path where the uploaded files are stored. | +| `INTERNAL_BACKEND_URL` | `http://localhost:8080` | no | The URL where the backend is accessible. | +| `GEOLITE_DB_PATH` | `data/GeoLite2-City.mmdb` | no | The path where the GeoLite2 database should be stored. | +| `CADDY_PORT` | `80` | no | The port on which Caddy should listen. Caddy is only active inside the Docker container. If you want to change the exposed port of the container then you sould change this variable. | +| `PORT` | `3000` | no | The port on which the frontend should listen. | +| `BACKEND_PORT` | `8080` | no | The port on which the backend should listen. | ## Contribute diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go index 82be7be..93e4296 100644 --- a/backend/internal/bootstrap/router_bootstrap.go +++ b/backend/internal/bootstrap/router_bootstrap.go @@ -35,7 +35,8 @@ func initRouter(db *gorm.DB, appConfigService *service.AppConfigService) { log.Fatalf("Unable to create email service: %s", err) } - auditLogService := service.NewAuditLogService(db, appConfigService, emailService) + geoLiteService := service.NewGeoLiteService() + auditLogService := service.NewAuditLogService(db, appConfigService, emailService, geoLiteService) jwtService := service.NewJwtService(appConfigService) webauthnService := service.NewWebAuthnService(db, jwtService, auditLogService, appConfigService) userService := service.NewUserService(db, jwtService, auditLogService) diff --git a/backend/internal/common/env_config.go b/backend/internal/common/env_config.go index 94f0ce4..e8a2eb7 100644 --- a/backend/internal/common/env_config.go +++ b/backend/internal/common/env_config.go @@ -14,6 +14,8 @@ type EnvConfigSchema struct { Port string `env:"BACKEND_PORT"` Host string `env:"HOST"` EmailTemplatesPath string `env:"EMAIL_TEMPLATES_PATH"` + MaxMindLicenseKey string `env:"MAXMIND_LICENSE_KEY"` + GeoLiteDBPath string `env:"GEOLITE_DB_PATH"` } var EnvConfig = &EnvConfigSchema{ @@ -24,6 +26,8 @@ var EnvConfig = &EnvConfigSchema{ Port: "8080", Host: "localhost", EmailTemplatesPath: "./email-templates", + MaxMindLicenseKey: "", + GeoLiteDBPath: "data/GeoLite2-City.mmdb", } func init() { diff --git a/backend/internal/service/audit_log_service.go b/backend/internal/service/audit_log_service.go index 09973a2..695f408 100644 --- a/backend/internal/service/audit_log_service.go +++ b/backend/internal/service/audit_log_service.go @@ -2,28 +2,27 @@ package service import ( userAgentParser "github.com/mileusna/useragent" - "github.com/oschwald/maxminddb-golang/v2" "github.com/stonith404/pocket-id/backend/internal/model" "github.com/stonith404/pocket-id/backend/internal/utils" "github.com/stonith404/pocket-id/backend/internal/utils/email" "gorm.io/gorm" "log" - "net/netip" ) type AuditLogService struct { db *gorm.DB appConfigService *AppConfigService emailService *EmailService + geoliteService *GeoLiteService } -func NewAuditLogService(db *gorm.DB, appConfigService *AppConfigService, emailService *EmailService) *AuditLogService { - return &AuditLogService{db: db, appConfigService: appConfigService, emailService: emailService} +func NewAuditLogService(db *gorm.DB, appConfigService *AppConfigService, emailService *EmailService, geoliteService *GeoLiteService) *AuditLogService { + return &AuditLogService{db: db, appConfigService: appConfigService, emailService: emailService, geoliteService: geoliteService} } // Create creates a new audit log entry in the database func (s *AuditLogService) Create(event model.AuditLogEvent, ipAddress, userAgent, userID string, data model.AuditLogData) model.AuditLog { - country, city, err := s.GetIpLocation(ipAddress) + country, city, err := s.geoliteService.GetLocationByIP(ipAddress) if err != nil { log.Printf("Failed to get IP location: %v\n", err) } @@ -97,29 +96,3 @@ func (s *AuditLogService) DeviceStringFromUserAgent(userAgent string) string { ua := userAgentParser.Parse(userAgent) return ua.Name + " on " + ua.OS + " " + ua.OSVersion } - -func (s *AuditLogService) GetIpLocation(ipAddress string) (country, city string, err error) { - db, err := maxminddb.Open("GeoLite2-City.mmdb") - if err != nil { - return "", "", err - } - defer db.Close() - - addr := netip.MustParseAddr(ipAddress) - - var record struct { - City struct { - Names map[string]string `maxminddb:"names"` - } `maxminddb:"city"` - Country struct { - Names map[string]string `maxminddb:"names"` - } `maxminddb:"country"` - } - - err = db.Lookup(addr).Decode(&record) - if err != nil { - return "", "", err - } - - return record.Country.Names["en"], record.City.Names["en"], nil -} diff --git a/backend/internal/service/geolite_service.go b/backend/internal/service/geolite_service.go new file mode 100644 index 0000000..d176652 --- /dev/null +++ b/backend/internal/service/geolite_service.go @@ -0,0 +1,142 @@ +package service + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "github.com/oschwald/maxminddb-golang/v2" + "github.com/stonith404/pocket-id/backend/internal/common" + "io" + "log" + "net/http" + "net/netip" + "os" + "path/filepath" + "time" +) + +type GeoLiteService struct{} + +// NewGeoLiteService initializes a new GeoLiteService instance and starts a goroutine to update the GeoLite2 City database. +func NewGeoLiteService() *GeoLiteService { + service := &GeoLiteService{} + + go func() { + if err := service.updateDatabase(); err != nil { + log.Printf("Failed to update GeoLite2 City database: %v\n", err) + } + }() + + return service +} + +// GetLocationByIP returns the country and city of the given IP address. +func (s *GeoLiteService) GetLocationByIP(ipAddress string) (country, city string, err error) { + db, err := maxminddb.Open(common.EnvConfig.GeoLiteDBPath) + if err != nil { + return "", "", err + } + defer db.Close() + + addr := netip.MustParseAddr(ipAddress) + + var record struct { + City struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"city"` + Country struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + } + + err = db.Lookup(addr).Decode(&record) + if err != nil { + return "", "", err + } + + return record.Country.Names["en"], record.City.Names["en"], nil +} + +// UpdateDatabase checks the age of the database and updates it if it's older than 14 days. +func (s *GeoLiteService) updateDatabase() error { + if s.isDatabaseUpToDate() { + log.Println("GeoLite2 City database is up-to-date.") + return nil + } + + log.Println("Updating GeoLite2 City database...") + + // Download and extract the database + downloadUrl := fmt.Sprintf( + "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz", + common.EnvConfig.MaxMindLicenseKey, + ) + // Download the database tar.gz file + resp, err := http.Get(downloadUrl) + if err != nil { + return fmt.Errorf("failed to download database: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download database, received HTTP %d", resp.StatusCode) + } + + // Extract the database file directly to the target path + if err := s.extractDatabase(resp.Body); err != nil { + return fmt.Errorf("failed to extract database: %w", err) + } + + log.Println("GeoLite2 City database successfully updated.") + return nil +} + +// isDatabaseUpToDate checks if the database file is older than 14 days. +func (s *GeoLiteService) isDatabaseUpToDate() bool { + info, err := os.Stat(common.EnvConfig.GeoLiteDBPath) + if err != nil { + // If the file doesn't exist, treat it as not up-to-date + return false + } + return time.Since(info.ModTime()) < 14*24*time.Hour +} + +// extractDatabase extracts the database file from the tar.gz archive directly to the target location. +func (s *GeoLiteService) extractDatabase(reader io.Reader) error { + gzr, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzr.Close() + + tarReader := tar.NewReader(gzr) + + // Iterate over the files in the tar archive + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read tar archive: %w", err) + } + + // Check if the file is the GeoLite2-City.mmdb file + if header.Typeflag == tar.TypeReg && filepath.Base(header.Name) == "GeoLite2-City.mmdb" { + outFile, err := os.Create(common.EnvConfig.GeoLiteDBPath) + if err != nil { + return fmt.Errorf("failed to create target database file: %w", err) + } + defer outFile.Close() + + // Write the file contents directly to the target location + if _, err := io.Copy(outFile, tarReader); err != nil { + return fmt.Errorf("failed to write database file: %w", err) + } + return nil + } + } + + return errors.New("GeoLite2-City.mmdb not found in archive") +} diff --git a/scripts/download-ip-database.sh b/scripts/download-ip-database.sh deleted file mode 100644 index 1971849..0000000 --- a/scripts/download-ip-database.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Check if the license key environment variable is set -if [ -z "$MAXMIND_LICENSE_KEY" ]; then - echo "Error: MAXMIND_LICENSE_KEY environment variable is not set." - echo "Please set it using 'export MAXMIND_LICENSE_KEY=your_license_key' and try again." - exit 1 -fi -echo $MAXMIND_LICENSE_KEY -# GeoLite2 City Database URL -URL="https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz" - -# Download directory -DOWNLOAD_DIR="./geolite2_db" -TARGET_PATH=./backend/GeoLite2-City.mmdb -mkdir -p $DOWNLOAD_DIR - -# Download the database -echo "Downloading GeoLite2 City database..." -curl -L -o "$DOWNLOAD_DIR/GeoLite2-City.tar.gz" "$URL" - -# Extract the downloaded file -echo "Extracting GeoLite2 City database..." -tar -xzf "$DOWNLOAD_DIR/GeoLite2-City.tar.gz" -C $DOWNLOAD_DIR --strip-components=1 - -mv "$DOWNLOAD_DIR/GeoLite2-City.mmdb" $TARGET_PATH - -# Clean up -rm -rf "$DOWNLOAD_DIR" - -echo "GeoLite2 City database downloaded and extracted to $TARGET_PATH" \ No newline at end of file