diff --git a/Pipfile b/Pipfile index 286f8d8..6c868ed 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,8 @@ verify_ssl = true [packages] foursquare = "*" ics = {editable = true,git = "https://github.com/C4ptainCrunch/ics.py.git",ref = "bd918ec7453a7cf73a906cdcc78bd88eb4bab71b"} +simplekml = "*" +pytz = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index a5dbbfb..26d7d95 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "09dd606d1eb6beaf39a922504397bc2a77c1063eed10c4d274caf1937dd80b89" + "sha256": "3a9e4292a546993c8c1f3f01a7183f293efeb295fd8fa776b213273b95279572" }, "pipfile-spec": 6, "requires": { @@ -24,10 +24,10 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "version": "==2019.3.9" + "version": "==2019.9.11" }, "chardet": { "hashes": [ @@ -38,11 +38,11 @@ }, "foursquare": { "hashes": [ - "sha256:339c95bd644cb18a622b7bdd2f5719354b46be8765bb545f5fa1862d91450e27", - "sha256:bd58725ec2a394cd01d2621099e91a6b15a49daad2d57ed732985b201da5021a" + "sha256:864004c812da8a117fb17c87d0f21f9c0c7460a6ca10b1016d5b21ef48cd5e10", + "sha256:99ed7c1ef49f7ac2f7fb0bee1e607c4ba2031b0204e1b13b1471cdaed60af6f4" ], "index": "pypi", - "version": "==1!2019.2.16" + "version": "==1!2019.9.11" }, "ics": { "editable": true, @@ -63,12 +63,27 @@ ], "version": "==2.8.0" }, + "pytz": { + "hashes": [ + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" + ], + "index": "pypi", + "version": "==2019.2" + }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "version": "==2.21.0" + "version": "==2.22.0" + }, + "simplekml": { + "hashes": [ + "sha256:30c121368ce1d73405721730bf766721e580cae6fbb7424884c734c89ec62ad7" + ], + "index": "pypi", + "version": "==1.3.1" }, "six": { "hashes": [ @@ -79,10 +94,10 @@ }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb", + "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47" ], - "version": "==1.24.2" + "version": "==1.25.5" } }, "develop": {} diff --git a/README.md b/README.md index 57f36a8..436dc5f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Foursquare Feeds -A python script that will generate an iCal (`.ics`) feed of your checkins on [Foursquare][4sq]/[Swarm][swarm]. If you set it up to save this file to a publicly-visible location on a webserver, and run the script regularly, you can subscribe to the feed in your favourite calendar application. +A python script that will generate iCal (`.ics`) or KML files of your checkins on [Foursquare][4sq]/[Swarm][swarm]. + +If you set it up to save the iCal file to a publicly-visible location on a webserver, and run the script regularly, you can subscribe to the feed in your favourite calendar application. + +A KML file can be loaded into a mapping application (such as Google Earth or +Maps) to view the checkins on a map. Foursquare [used to have such feeds][feeds] but they've stopped working for me. [I wrote a bit about this.][blog] @@ -32,9 +37,11 @@ or [pip](https://pip.pypa.io/en/stable/): ### 3. Set up config file -Copy `config_example.ini` to `config.ini`. Change the `IcsFilepath` to wherever you want your file to be saved. +Copy `config_example.ini` to `config.ini`. -To get the `AccessToken` for your Foursquare app, you will have to go through the laborious procedure in step 4... +Change the `IcsFilepath` and `KmlFilepath` to wherever you want your files to be saved. + +To get the `AccessToken` for your Foursquare app, you will have to go through the sometimes laborious procedure in step 4... ### 4. Get an access token @@ -93,7 +100,7 @@ Enter this in your `config.ini`. ## Run the script -You're ready to go: +Generate a `.ics` file: $ ./generate_feeds.py @@ -104,6 +111,9 @@ If the file is generated in a location on your website that's publicly-visible, Note that the file might contain private checkins or information you don't want to be public. In which case, it's probably best to make the name of any such publicly-readable file very obscure. +To generate a `.kml` file, see the `kind` option below. + + ### Script options #### `--all` @@ -118,6 +128,18 @@ Depending on how many checkins you have you might only want to run it with `--all` the first time and, once that's imported into a calendar application, subsequently only fetch recent checkins. +### `-k` or `--kind` + +By default the script generates an iCal `.ics` file. Or, use this option to +specify an `.ics` file or a `.kml` file: + +```bash +$ ./generate_feeds.py -k ics +$ ./generate_feeds.py -k kml +$ ./generate_feeds.py --kind=ics +$ ./generate_feeds.py --kind=kml +``` + #### `-v` or `--verbose` By default the script will only output text if something goes wrong. To get diff --git a/config_example.ini b/config_example.ini index 1b5aabc..59980a2 100644 --- a/config_example.ini +++ b/config_example.ini @@ -6,3 +6,4 @@ AccessToken=YourAccessTokenHere IcsFilepath=./foursquare.ics +KmlFilepath=./foursquare.kml diff --git a/generate_feeds.py b/generate_feeds.py index 6730a8d..5b4953b 100755 --- a/generate_feeds.py +++ b/generate_feeds.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 import argparse import configparser +from datetime import datetime import logging import os +import pytz +from xml.sax.saxutils import escape as xml_escape import foursquare from ics import Calendar, Event +import simplekml logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -14,6 +18,9 @@ logger = logging.getLogger(__name__) current_dir = os.path.realpath(os.path.dirname(__file__)) CONFIG_FILE = os.path.join(current_dir, "config.ini") +# The kinds of file we can generate: +VALID_KINDS = ["ics", "kml"] + class FeedGenerator: @@ -40,23 +47,27 @@ class FeedGenerator: self.api_access_token = config.get("Foursquare", "AccessToken") self.ics_filepath = config.get("Local", "IcsFilepath") + self.kml_filepath = config.get("Local", "KmlFilepath") - def generate(self): + def generate(self, kind="ics"): "Call this to fetch the data from the API and generate the file." + if kind not in VALID_KINDS: + raise ValueError(f"kind should be one of {', '.join(VALID_KINDS)}.") + if self.fetch == "all": checkins = self._get_all_checkins() else: checkins = self._get_recent_checkins() plural = "" if len(checkins) == 1 else "s" - logger.info("Fetched {} checkin{} from the API".format(len(checkins), plural)) + logger.info(f"Fetched {checkins} checkin{plural} from the API") - calendar = self._generate_calendar(checkins) + if kind == "ics": + filepath = self._generate_ics_file(checkins) + elif kind == "kml": + filepath = self._generate_kml_file(checkins) - with open(self.ics_filepath, "w") as f: - f.writelines(calendar) - - logger.info("Generated calendar file {}".format(self.ics_filepath)) + logger.info(f"Generated file {filepath}") exit(0) @@ -80,9 +91,9 @@ class FeedGenerator: # First time, set the correct total: total_checkins = results["checkins"]["count"] plural = "" if total_checkins == 1 else "s" - logger.debug("{} checkin{} to fetch".format(total_checkins, plural)) + logger.debug(f"{total_checkins} checkin{plural} to fetch") - logger.debug("Fetched {}-{}".format((offset + 1), (offset + 250))) + logger.debug(f"Fetched {offset+1}-{offset+250}") checkins += results["checkins"]["items"] offset += 250 @@ -102,20 +113,34 @@ class FeedGenerator: params={"limit": 250, "offset": offset, "sort": "newestfirst"} ) except foursquare.FoursquareException as err: - logger.error( - "Error getting checkins, with offset of {}: {}".format(offset, err) - ) + logger.error(f"Error getting checkins, with offset of {offset}: {err}") exit(1) - def _get_user_url(self): - "Returns the Foursquare URL for the authenticated user." + def _get_user(self): + "Returns details about the authenticated user." try: user = self.client.users() except foursquare.FoursquareException as err: - logger.error("Error getting user: {}".format(err)) + logger.error(f"Error getting user: {err}") exit(1) - return user["user"]["canonicalUrl"] + return user["user"] + + def _generate_ics_file(self, checkins): + """Supplied with a list of checkin data from the API, generates + and saves a .ics file. + + Returns the filepath of the saved file. + + Keyword arguments: + checkins -- A list of dicts, each one data about a single checkin. + """ + calendar = self._generate_calendar(checkins) + + with open(self.ics_filepath, "w") as f: + f.writelines(calendar) + + return self.ics_filepath def _generate_calendar(self, checkins): """Supplied with a list of checkin data from the API, generates @@ -124,7 +149,7 @@ class FeedGenerator: Keyword arguments: checkins -- A list of dicts, each one data about a single checkin. """ - user_url = self._get_user_url() + user = self._get_user() c = Calendar() @@ -133,15 +158,16 @@ class FeedGenerator: # I had some checkins with no data other than # id, createdAt and source. continue + venue_name = checkin["venue"]["name"] tz_offset = self._get_checkin_timezone(checkin) e = Event() - e.name = "@ {}".format(venue_name) + e.name = f"@ {venue_name}" e.location = venue_name - e.url = "{}/checkin/{}".format(user_url, checkin["id"]) - e.uid = "{}@foursquare.com".format(checkin["id"]) + e.url = f"{user['canonicalUrl']}/checkin/{checkin['id']}" + e.uid = f"{checkin['id']}@foursquare.com" e.begin = checkin["createdAt"] # Use the 'shout', if any, and the timezone offset in the @@ -149,7 +175,7 @@ class FeedGenerator: description = [] if "shout" in checkin and len(checkin["shout"]) > 0: description = [checkin["shout"]] - description.append("Timezone offset: {}".format(tz_offset)) + description.append(f"Timezone offset: {tz_offset}") e.description = "\n".join(description) # Use the venue_name and the address, if any, for the location. @@ -158,13 +184,87 @@ class FeedGenerator: loc = checkin["venue"]["location"] if "formattedAddress" in loc and len(loc["formattedAddress"]) > 0: address = ", ".join(loc["formattedAddress"]) - location = "{}, {}".format(location, address) + location = f"{location}, {address}" e.location = location c.events.add(e) return c + def _generate_kml_file(self, checkins): + """Supplied with a list of checkin data from the API, generates + and saves a kml file. + + Returns the filepath of the saved file. + + Keyword arguments: + checkins -- A list of dicts, each one data about a single checkin. + """ + user = self._get_user() + + kml = simplekml.Kml() + + # The original Foursquare files had a Folder with name and + # description like this, so: + user_name = f"{user['firstName']} {user['lastName']}" + name = f"foursquare checkin history for {user_name}" + fol = kml.newfolder(name=name, description=name) + + for checkin in checkins: + if "venue" not in checkin: + # I had some checkins with no data other than + # id, createdAt and source. + continue + + venue_name = checkin["venue"]["name"] + tz_offset = self._get_checkin_timezone(checkin) + url = f'https://foursquare.com/v/{checkin["venue"]["id"]}' + + description = [f'@{venue_name}'] + if "shout" in checkin and len(checkin["shout"]) > 0: + description.append('"{}"'.format(checkin["shout"])) + description.append(f"Timezone offset: {tz_offset}") + + coords = [ + ( + checkin["venue"]["location"]["lng"], + checkin["venue"]["location"]["lat"], + ) + ] + + visibility = 0 if "private" in checkin else 1 + + pnt = fol.newpoint( + name=venue_name, + description="".format('\n'.join(description)), + coords=coords, + visibility=visibility, + # Both of these were set like this in Foursquare's original KML: + altitudemode=simplekml.AltitudeMode.relativetoground, + extrude=1, + ) + + # Foursquare's KML feeds had 'updated' and 'published' elements + # in the Placemark, but I don't *think* those are standard, so: + pnt.timestamp.when = ( + datetime.utcfromtimestamp(checkin["createdAt"]) + .replace(tzinfo=pytz.utc) + .isoformat() + ) + + # Use the address, if any: + if "location" in checkin["venue"]: + loc = checkin["venue"]["location"] + if "formattedAddress" in loc and len(loc["formattedAddress"]) > 0: + address = ", ".join(loc["formattedAddress"]) + # While simplexml escapes other strings, it threw a wobbly + # over '&' in addresses, so escape them: + pnt.address = xml_escape(address) + + kml.save(self.kml_filepath) + + return self.kml_filepath + def _get_checkin_timezone(self, checkin): """Given a checkin from the API, returns a string representing the timezone offset of that checkin. @@ -192,7 +292,7 @@ class FeedGenerator: symbol = "" # e.g. '+01:00' or '-08.00' - return "{}{}".format(symbol, offset).replace(".", ":") + return f"{symbol}{offset}".replace(".", ":") if __name__ == "__main__": @@ -209,6 +309,15 @@ if __name__ == "__main__": default=False, ) + parser.add_argument( + "-k", + "--kind", + action="store", + help="Either ics (default) or kml", + required=False, + type=str, + ) + parser.add_argument( "-v", "--verbose", @@ -231,8 +340,16 @@ if __name__ == "__main__": else: to_fetch = "recent" + if args.kind: + if args.kind in VALID_KINDS: + kind = args.kind + else: + raise ValueError(f"kind should be one of {', '.join(VALID_KINDS)}.") + else: + kind = "ics" + generator = FeedGenerator(fetch=to_fetch) - generator.generate() + generator.generate(kind=kind) exit(0)