Merge pull request #4 from philgyford/add-kml-3

Add support for generating KML files

Pipfile

@@ -8,6 +8,8 @@ verify_ssl = true
 [packages]
 foursquare = "*"
 ics = {editable = true,git = "https://github.com/C4ptainCrunch/ics.py.git",ref = "bd918ec7453a7cf73a906cdcc78bd88eb4bab71b"}
+simplekml = "*"
+pytz = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "09dd606d1eb6beaf39a922504397bc2a77c1063eed10c4d274caf1937dd80b89"
+            "sha256": "3a9e4292a546993c8c1f3f01a7183f293efeb295fd8fa776b213273b95279572"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,10 +24,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
+                "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
-                "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
+                "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
             ],
-            "version": "==2019.3.9"
+            "version": "==2019.9.11"
         },
         "chardet": {
             "hashes": [
@@ -38,11 +38,11 @@
         },
         "foursquare": {
             "hashes": [
-                "sha256:339c95bd644cb18a622b7bdd2f5719354b46be8765bb545f5fa1862d91450e27",
+                "sha256:864004c812da8a117fb17c87d0f21f9c0c7460a6ca10b1016d5b21ef48cd5e10",
-                "sha256:bd58725ec2a394cd01d2621099e91a6b15a49daad2d57ed732985b201da5021a"
+                "sha256:99ed7c1ef49f7ac2f7fb0bee1e607c4ba2031b0204e1b13b1471cdaed60af6f4"
             ],
             "index": "pypi",
-            "version": "==1!2019.2.16"
+            "version": "==1!2019.9.11"
         },
         "ics": {
             "editable": true,
@@ -63,12 +63,27 @@
             ],
             "version": "==2.8.0"
         },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "index": "pypi",
+            "version": "==2019.2"
+        },
         "requests": {
             "hashes": [
-                "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
-                "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
             ],
-            "version": "==2.21.0"
+            "version": "==2.22.0"
+        },
+        "simplekml": {
+            "hashes": [
+                "sha256:30c121368ce1d73405721730bf766721e580cae6fbb7424884c734c89ec62ad7"
+            ],
+            "index": "pypi",
+            "version": "==1.3.1"
         },
         "six": {
             "hashes": [
@@ -79,10 +94,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
+                "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb",
-                "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
+                "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47"
             ],
-            "version": "==1.24.2"
+            "version": "==1.25.5"
         }
     },
     "develop": {}

README.md

@@ -1,6 +1,11 @@
 # Foursquare Feeds
 
-A python script that will generate an iCal (`.ics`) feed of your checkins on [Foursquare][4sq]/[Swarm][swarm]. If you set it up to save this file to a publicly-visible location on a webserver, and run the script regularly, you can subscribe to the feed in your favourite calendar application.
+A python script that will generate iCal (`.ics`) or KML files of your checkins on [Foursquare][4sq]/[Swarm][swarm].
+
+If you set it up to save the iCal file to a publicly-visible location on a webserver, and run the script regularly, you can subscribe to the feed in your favourite calendar application.
+
+A KML file can be loaded into a mapping application (such as Google Earth or
+Maps) to view the checkins on a map.
 
 Foursquare [used to have such feeds][feeds] but they've stopped working for me.
 [I wrote a bit about this.][blog]
@@ -32,9 +37,11 @@ or [pip](https://pip.pypa.io/en/stable/):
 
 ### 3. Set up config file
 
-Copy `config_example.ini` to `config.ini`. Change the `IcsFilepath` to wherever you want your file to be saved.
+Copy `config_example.ini` to `config.ini`.
 
-To get the `AccessToken` for your Foursquare app, you will have to go through the laborious procedure in step 4...
+Change the `IcsFilepath` and `KmlFilepath` to wherever you want your files to be saved.
+
+To get the `AccessToken` for your Foursquare app, you will have to go through the sometimes laborious procedure in step 4...
 
 
 ### 4. Get an access token
@@ -93,7 +100,7 @@ Enter this in your `config.ini`.
 
 ## Run the script
 
-You're ready to go:
+Generate a `.ics` file:
 
     $ ./generate_feeds.py
 
@@ -104,6 +111,9 @@ If the file is generated in a location on your website that's publicly-visible,
 
 Note that the file might contain private checkins or information you don't want to be public. In which case, it's probably best to make the name of any such publicly-readable file very obscure.
 
+To generate a `.kml` file, see the `kind` option below.
+
+
 ### Script options
 
 #### `--all`
@@ -118,6 +128,18 @@ Depending on how many checkins you have you might only want to run it with
 `--all` the first time and, once that's imported into a calendar application,
 subsequently only fetch recent checkins.
 
+### `-k` or `--kind`
+
+By default the script generates an iCal `.ics` file. Or, use this option to
+specify an `.ics` file or a `.kml` file:
+
+```bash
+$ ./generate_feeds.py -k ics
+$ ./generate_feeds.py -k kml
+$ ./generate_feeds.py --kind=ics
+$ ./generate_feeds.py --kind=kml
+```
+
 #### `-v` or `--verbose`
 
 By default the script will only output text if something goes wrong. To get

config_example.ini

@@ -6,3 +6,4 @@ AccessToken=YourAccessTokenHere
 
 IcsFilepath=./foursquare.ics
 
+KmlFilepath=./foursquare.kml

generate_feeds.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
 import argparse
 import configparser
+from datetime import datetime
 import logging
 import os
+import pytz
+from xml.sax.saxutils import escape as xml_escape
 
 import foursquare
 from ics import Calendar, Event
+import simplekml
 
 logging.basicConfig(level=logging.INFO, format="%(message)s")
 logger = logging.getLogger(__name__)
@@ -14,6 +18,9 @@ logger = logging.getLogger(__name__)
 current_dir = os.path.realpath(os.path.dirname(__file__))
 CONFIG_FILE = os.path.join(current_dir, "config.ini")
 
+# The kinds of file we can generate:
+VALID_KINDS = ["ics", "kml"]
+
 
 class FeedGenerator:
 
@@ -40,23 +47,27 @@ class FeedGenerator:
 
         self.api_access_token = config.get("Foursquare", "AccessToken")
         self.ics_filepath = config.get("Local", "IcsFilepath")
+        self.kml_filepath = config.get("Local", "KmlFilepath")
 
-    def generate(self):
+    def generate(self, kind="ics"):
         "Call this to fetch the data from the API and generate the file."
+        if kind not in VALID_KINDS:
+            raise ValueError(f"kind should be one of {', '.join(VALID_KINDS)}.")
+
         if self.fetch == "all":
             checkins = self._get_all_checkins()
         else:
             checkins = self._get_recent_checkins()
 
         plural = "" if len(checkins) == 1 else "s"
-        logger.info("Fetched {} checkin{} from the API".format(len(checkins), plural))
+        logger.info(f"Fetched {checkins} checkin{plural} from the API")
 
-        calendar = self._generate_calendar(checkins)
+        if kind == "ics":
+            filepath = self._generate_ics_file(checkins)
+        elif kind == "kml":
+            filepath = self._generate_kml_file(checkins)
 
-        with open(self.ics_filepath, "w") as f:
+        logger.info(f"Generated file {filepath}")
-            f.writelines(calendar)
-
-        logger.info("Generated calendar file {}".format(self.ics_filepath))
 
         exit(0)
 
@@ -80,9 +91,9 @@ class FeedGenerator:
                 # First time, set the correct total:
                 total_checkins = results["checkins"]["count"]
                 plural = "" if total_checkins == 1 else "s"
-                logger.debug("{} checkin{} to fetch".format(total_checkins, plural))
+                logger.debug(f"{total_checkins} checkin{plural} to fetch")
 
-            logger.debug("Fetched {}-{}".format((offset + 1), (offset + 250)))
+            logger.debug(f"Fetched {offset+1}-{offset+250}")
 
             checkins += results["checkins"]["items"]
             offset += 250
@@ -102,20 +113,34 @@ class FeedGenerator:
                 params={"limit": 250, "offset": offset, "sort": "newestfirst"}
             )
         except foursquare.FoursquareException as err:
-            logger.error(
+            logger.error(f"Error getting checkins, with offset of {offset}: {err}")
-                "Error getting checkins, with offset of {}: {}".format(offset, err)
-            )
             exit(1)
 
-    def _get_user_url(self):
+    def _get_user(self):
-        "Returns the Foursquare URL for the authenticated user."
+        "Returns details about the authenticated user."
         try:
             user = self.client.users()
         except foursquare.FoursquareException as err:
-            logger.error("Error getting user: {}".format(err))
+            logger.error(f"Error getting user: {err}")
             exit(1)
 
-        return user["user"]["canonicalUrl"]
+        return user["user"]
+
+    def _generate_ics_file(self, checkins):
+        """Supplied with a list of checkin data from the API, generates
+        and saves a .ics file.
+
+        Returns the filepath of the saved file.
+
+        Keyword arguments:
+        checkins -- A list of dicts, each one data about a single checkin.
+        """
+        calendar = self._generate_calendar(checkins)
+
+        with open(self.ics_filepath, "w") as f:
+            f.writelines(calendar)
+
+        return self.ics_filepath
 
     def _generate_calendar(self, checkins):
         """Supplied with a list of checkin data from the API, generates
@@ -124,7 +149,7 @@ class FeedGenerator:
         Keyword arguments:
         checkins -- A list of dicts, each one data about a single checkin.
         """
-        user_url = self._get_user_url()
+        user = self._get_user()
 
         c = Calendar()
 
@@ -133,15 +158,16 @@ class FeedGenerator:
                 # I had some checkins with no data other than
                 # id, createdAt and source.
                 continue
+
             venue_name = checkin["venue"]["name"]
             tz_offset = self._get_checkin_timezone(checkin)
 
             e = Event()
 
-            e.name = "@ {}".format(venue_name)
+            e.name = f"@ {venue_name}"
             e.location = venue_name
-            e.url = "{}/checkin/{}".format(user_url, checkin["id"])
+            e.url = f"{user['canonicalUrl']}/checkin/{checkin['id']}"
-            e.uid = "{}@foursquare.com".format(checkin["id"])
+            e.uid = f"{checkin['id']}@foursquare.com"
             e.begin = checkin["createdAt"]
 
             # Use the 'shout', if any, and the timezone offset in the
@@ -149,7 +175,7 @@ class FeedGenerator:
             description = []
             if "shout" in checkin and len(checkin["shout"]) > 0:
                 description = [checkin["shout"]]
-            description.append("Timezone offset: {}".format(tz_offset))
+            description.append(f"Timezone offset: {tz_offset}")
             e.description = "\n".join(description)
 
             # Use the venue_name and the address, if any, for the location.
@@ -158,13 +184,87 @@ class FeedGenerator:
                 loc = checkin["venue"]["location"]
                 if "formattedAddress" in loc and len(loc["formattedAddress"]) > 0:
                     address = ", ".join(loc["formattedAddress"])
-                    location = "{}, {}".format(location, address)
+                    location = f"{location}, {address}"
             e.location = location
 
             c.events.add(e)
 
         return c
 
+    def _generate_kml_file(self, checkins):
+        """Supplied with a list of checkin data from the API, generates
+        and saves a kml file.
+
+        Returns the filepath of the saved file.
+
+        Keyword arguments:
+        checkins -- A list of dicts, each one data about a single checkin.
+        """
+        user = self._get_user()
+
+        kml = simplekml.Kml()
+
+        # The original Foursquare files had a Folder with name and
+        # description like this, so:
+        user_name = f"{user['firstName']} {user['lastName']}"
+        name = f"foursquare checkin history for {user_name}"
+        fol = kml.newfolder(name=name, description=name)
+
+        for checkin in checkins:
+            if "venue" not in checkin:
+                # I had some checkins with no data other than
+                # id, createdAt and source.
+                continue
+
+            venue_name = checkin["venue"]["name"]
+            tz_offset = self._get_checkin_timezone(checkin)
+            url = f'https://foursquare.com/v/{checkin["venue"]["id"]}'
+
+            description = [f'@<a href="{url}">{venue_name}</a>']
+            if "shout" in checkin and len(checkin["shout"]) > 0:
+                description.append('"{}"'.format(checkin["shout"]))
+            description.append(f"Timezone offset: {tz_offset}")
+
+            coords = [
+                (
+                    checkin["venue"]["location"]["lng"],
+                    checkin["venue"]["location"]["lat"],
+                )
+            ]
+
+            visibility = 0 if "private" in checkin else 1
+
+            pnt = fol.newpoint(
+                name=venue_name,
+                description="<![CDATA[{}]]>".format('\n'.join(description)),
+                coords=coords,
+                visibility=visibility,
+                # Both of these were set like this in Foursquare's original KML:
+                altitudemode=simplekml.AltitudeMode.relativetoground,
+                extrude=1,
+            )
+
+            # Foursquare's KML feeds had 'updated' and 'published' elements
+            # in the Placemark, but I don't *think* those are standard, so:
+            pnt.timestamp.when = (
+                datetime.utcfromtimestamp(checkin["createdAt"])
+                .replace(tzinfo=pytz.utc)
+                .isoformat()
+            )
+
+            # Use the address, if any:
+            if "location" in checkin["venue"]:
+                loc = checkin["venue"]["location"]
+                if "formattedAddress" in loc and len(loc["formattedAddress"]) > 0:
+                    address = ", ".join(loc["formattedAddress"])
+                    # While simplexml escapes other strings, it threw a wobbly
+                    # over '&' in addresses, so escape them:
+                    pnt.address = xml_escape(address)
+
+        kml.save(self.kml_filepath)
+
+        return self.kml_filepath
+
     def _get_checkin_timezone(self, checkin):
         """Given a checkin from the API, returns a string representing the
         timezone offset of that checkin.
@@ -192,7 +292,7 @@ class FeedGenerator:
             symbol = ""
 
         # e.g. '+01:00' or '-08.00'
-        return "{}{}".format(symbol, offset).replace(".", ":")
+        return f"{symbol}{offset}".replace(".", ":")
 
 
 if __name__ == "__main__":
@@ -209,6 +309,15 @@ if __name__ == "__main__":
         default=False,
     )
 
+    parser.add_argument(
+        "-k",
+        "--kind",
+        action="store",
+        help="Either ics (default) or kml",
+        required=False,
+        type=str,
+    )
+
     parser.add_argument(
         "-v",
         "--verbose",
@@ -231,8 +340,16 @@ if __name__ == "__main__":
     else:
         to_fetch = "recent"
 
+    if args.kind:
+        if args.kind in VALID_KINDS:
+            kind = args.kind
+        else:
+            raise ValueError(f"kind should be one of {', '.join(VALID_KINDS)}.")
+    else:
+        kind = "ics"
+
     generator = FeedGenerator(fetch=to_fetch)
 
-    generator.generate()
+    generator.generate(kind=kind)
 
     exit(0)