From 9c92963cb4cf320a2afadf1c5634f84635de9d06 Mon Sep 17 00:00:00 2001 From: CountablyInfinite Date: Wed, 18 Nov 2020 12:23:25 +0100 Subject: [PATCH 1/5] initial commit, Added functionality to download more than 500 workouts, added feature to format output to be compatible with Garmin Connect. --- README.md | 55 +++++++++++------------- export.py | 37 +++++++++++----- lib/endomondo.py | 110 ++++++++++++++++++++++++++++++++++++++++------- lib/tcx.py | 17 ++++++-- requirements.txt | 2 +- 5 files changed, 161 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 9b971fe..dda1d0c 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,42 @@ -Endomondo Export -================ +# Endomondo Export -Export a user's most recent Endomondo workouts as TCX files. +This is an extended fork of https://github.com/yannickcarer/endomondo-export. Since Endomondo is shutting down it's service with the end of 2020 (what a year), i was searching for a way to export all my workouts (>1000) to Garmin Connect. yannickcarers script works fine (despite beeing python 2.x), but you can not export more than ~500 workouts since the server will timeout roughly around 500 exports. I added a feature than downloads the last 15 years as chunks of 500 trainings and also added the option to format the output for Garmin Connects import function, which needs an extra Wrapper around trackpoints and only allows "Running", "Biking" and "Other" as activity names. +## Usage -Usage ------ +The script export.py may be used to backup the complete workout history. You will be asked for your Endomondo email and password, followed by the number of workouts you want to export (leave empty to export the last 15 years) and if you want to format the export to be compatible with Garmin Connect. -The script export.py may be used to backup the complete workout history: +**Important: If you have done more than 500 workouts in a year (congratulations!) you will run into problems using my script!** - python export.py +```shell +python export.py +Endomondo: Export the most recent n workouts (or ALL) as TCX files. +Email: mymail@yourdomain.com +Password: +Maximum number of workouts n (press Enter to ignore): 5 +Format for Garmin Connect? (press Enter to ignore): Y +Files will be formatted to be compatible with Garmin Connects import function. +fetched latest 5 workouts +writing export/20201113_1648626623.tcx, Other, 1 trackpoints +writing export/20201112_1647888548.tcx, Biking, 693 trackpoints +writing export/20201110_1647453368.tcx, Running, 1319 trackpoints +... +``` -You will be asked for your Endomondo email and password, and then the script will do the rest. +## Requirements -Requirements ------------- - - Python 2.6+ - lxml - requests -Installing ----------- - -To set up the requirements for this project, you can install the dependencies by pip. - -First, it's highly recommended to set up a virtualenv: - - virtualenv venv --distribute - source ./venv/bin/activate - -Then install the requirements: - - pip install -r requirements - -And you're set! +## Installing +```shell +pip install -r requirements +``` -Authors -------- +## Credit -This script was created [@yannickcarer](https://github.com/yannickcarer), with some updates by [@mikedory](https://github.com/mikedory). +This script was created [@yannickcarer](https://github.com/yannickcarer), with some updates by [@mikedory](https://github.com/mikedory) and some minor improvements of [@countablyinfinite](https://github.com/countablyinfinite) diff --git a/export.py b/export.py index e12fae0..52bf34e 100644 --- a/export.py +++ b/export.py @@ -5,7 +5,7 @@ import re import getpass import sys -import os +import os, datetime # create a somewhat useful filename for the specified workout @@ -30,7 +30,7 @@ def create_directory(directory): # create the TCX file for the specified workout -def create_tcx_file(workout): +def create_tcx_file(workout, garmin): directory_name = 'export' activity = workout.get_activity() name = create_filename(workout) @@ -38,7 +38,7 @@ def create_tcx_file(workout): filename = os.path.join(directory_name, name) print "writing %s, %s, %s trackpoints" % (filename, activity.sport, len(activity.trackpoints)) - writer = tcx.Writer() + writer = tcx.Writer(garmin) tcxfile = writer.write(activity) if tcxfile: with open(filename, 'w') as f: @@ -47,17 +47,34 @@ def create_tcx_file(workout): def main(): try: - print "Endomondo: export most recent workouts as TCX files" - - email = raw_input("Email: ") + print "Endomondo: export most recent n workouts (or ALL) as TCX files." + mail = raw_input("Email: ") password = getpass.getpass() - maximum_workouts = raw_input("Maximum number of workouts (press Enter to ignore)") - endomondo = Endomondo(email, password) + maximum_workouts = raw_input("Maximum number of workouts n (press Enter to ignore): ") + garmin = raw_input("Format for Garmin Connect? (press Enter to ignore): ") + endomondo = Endomondo(mail, password, garmin) + if garmin: + print "Files will be formatted to be compatible with Garmin Connects import function." + + if not maximum_workouts: + days_per_year = 365.24 + maximum_workouts = 500 + print "Downloading workouts from the last 15 years in chunks. (If you logged more than 500 workouts per year, this won't work)" + for years in range(0,20): + before=(datetime.datetime.now()-datetime.timedelta(days=(days_per_year*years))) + after=before-datetime.timedelta(days=(days_per_year*(1))) + print "Chunk before:" +str(before) + print "Chunk after:" +str(after) + workouts = endomondo.get_workouts(maximum_workouts, before, after) + for workout in workouts: + create_tcx_file(workout,garmin) + print "done." + return 0 - workouts = endomondo.get_workouts(maximum_workouts) + workouts = endomondo.get_workouts(maximum_workouts, before=None, after=None) print "fetched latest", len(workouts), "workouts" for workout in workouts: - create_tcx_file(workout) + create_tcx_file(workout, garmin) print "done." return 0 diff --git a/lib/endomondo.py b/lib/endomondo.py index 557f956..a78b66c 100644 --- a/lib/endomondo.py +++ b/lib/endomondo.py @@ -5,7 +5,8 @@ import uuid import socket import datetime - +import pytz +import sys def to_datetime(v): return datetime.datetime.strptime(v, "%Y-%m-%d %H:%M:%S %Z") @@ -29,6 +30,59 @@ def to_meters(v): v *= 1000 return v +SPORTS_GARMIN = { + 0: 'Running', + 1: 'Biking', + 2: 'Biking', + 3: 'Biking', + 4: 'Other', + 5: 'Other', + 6: 'Other', + 7: 'Other', + 8: 'Other', + 9: 'Other', + 10: 'Other', + 11: 'Other', + 12: 'Other', + 13: 'Other', + 14: 'Other', + 15: 'Other', + 17: 'Other', + 18: 'Other', + 19: 'Other', + 20: 'Other', + 21: 'Other', + 22: 'Other', + 23: 'Other', + 24: 'Other', + 25: 'Other', + 26: 'Other', + 27: 'Other', + 28: 'Other', + 29: 'Other', + 30: 'Other', + 31: 'Other', + 32: 'Other', + 33: 'Other', + 34: 'Other', + 35: 'Other', + 36: 'Other', + 37: 'Other', + 38: 'Other', + 39: 'Other', + 40: 'Other', + 41: 'Other', + 42: 'Other', + 43: 'Other', + 44: 'Other', + 45: 'Other', + 46: 'Other', + 47: 'Other', + 48: 'Other', + 49: 'Other', + 50: 'Other' +} + SPORTS = { 0: 'Running', 1: 'Cycling, transport', @@ -89,19 +143,20 @@ class Endomondo: os_version = "2.2" model = "M" - def __init__(self, email=None, password=None): + def __init__(self, email=None, password=None, garmin=None): self.auth_token = None self.request = requests.session() self.request.headers['User-Agent'] = self.get_user_agent() if email and password: self.auth_token = self.request_auth_token(email, password) + self.garmin = garmin def get_user_agent(self): """HTTP User-Agent""" return "Dalvik/1.4.0 (Linux; U; %s %s; %s Build/GRI54)" % (self.os, self.os_version, self.model) def get_device_id(self): - return str(uuid.uuid5(uuid.NAMESPACE_DNS, socket.gethostname())) + return "ebb1083f-7ead-5e11-9847-86fc1d77348e"#str(uuid.uuid5(uuid.NAMESPACE_DNS, socket.gethostname())) def request_auth_token(self, email, password): @@ -173,22 +228,32 @@ def call(self, url, format, params={}): return self.parse_json(r) return r - def get_workouts(self, max_results=40): - """Get the most recent workouts""" - if not max_results: - max_results = 100000000 - json = self.call('api/workout/list', 'json', - {'maxResults': max_results}) - return [EndomondoWorkout(self, w) for w in json] + def get_workouts(self, max_results, before, after): + """Get workouts""" + params = {'maxResults': max_results} + if before: + params.update({'after': self.to_endomondo_time(after)}) + params.update({'before': self.to_endomondo_time(before)}) + + json = self.call('api/workout/list', 'json', params) + return [EndomondoWorkout(self, w, self.garmin) for w in json] + + def to_endomondo_time(self,time): + return time.strftime("%Y-%m-%d %H:%M:%S UTC") + + + def to_python_time(self, endomondo_time): + return datetime.datetime.strptime(endomondo_time, "%Y-%m-%d %H:%M:%S UTC").replace(tzinfo=pytz.utc) class EndomondoWorkout: """Endomondo Workout wrapper""" - def __init__(self, parent, properties): + def __init__(self, parent, properties, garmin): self.parent = parent self.properties = properties self.activity = None + self.garmin = garmin # dict wrapper def __getattr__(self, name): @@ -196,7 +261,10 @@ def __getattr__(self, name): if name in self.properties: value = self.properties[name] if name == 'sport': - value = SPORTS.get(value, 'Other') + if self.garmin: + value = SPORTS_GARMIN.get(value, 'Other') + else: + value = SPORTS.get(value, 'Other') elif name == 'start_time': value = to_datetime(value) return value @@ -212,9 +280,16 @@ def get_activity(self): # the 1st line is activity details data = lines[0].split(";") - start_time = to_datetime(data[6]) + try: + start_time = to_datetime(data[6]) + except Exception as e: + print "start time" + sys.exit() self.activity = tcx.Activity() - self.activity.sport = SPORTS.get(int(data[5]), "Other") + if self.garmin: + self.activity.sport = SPORTS_GARMIN.get(int(data[5]), "Other") + else: + self.activity.sport = SPORTS.get(int(data[5]), "Other") self.activity.start_time = start_time self.activity.notes = self.notes @@ -235,8 +310,11 @@ def get_activity(self): for line in lines[1:]: data = line.split(";") if len(data) >= 9: - w = tcx.Trackpoint() - w.timestamp = to_datetime(data[0]) + w = tcx.Trackpoint() + try: + w.timestamp = to_datetime(data[0]) + except Exception as e: + w.timestamp=start_time w.latitude = to_float(data[2]) w.longitude = to_float(data[3]) w.altitude_meters = to_float(data[6]) diff --git a/lib/tcx.py b/lib/tcx.py index 763bfcd..338efba 100644 --- a/lib/tcx.py +++ b/lib/tcx.py @@ -40,6 +40,9 @@ def __init__(self): class Writer: + def __init__(self, garmin): + self.garmin = garmin + TCD_NAMESPACE = "http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2" XML_SCHEMA_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" @@ -91,6 +94,7 @@ def add_heart_rate(self, element, name, value): def add_lap(self, element, activity, lap): elem = self.create_element(element, "Lap") + elem.set("StartTime", self.time_to_s(lap.start_time)) self.add_property(elem, "TotalTimeSeconds", lap.total_time_seconds) @@ -102,10 +106,15 @@ def add_lap(self, element, activity, lap): self.add_property(elem, "Intensity", lap.intensity) self.add_property(elem, "Cadence", lap.cadence) self.add_property(elem, "TriggerMethod", lap.trigger_method) - - # Add trackpoints - for w in activity.trackpoints: - self.add_trackpoint(elem, w) + # Garmin needs an extra Track wrapper + if self.garmin: + track = self.create_element(elem, "Track") + # Add trackpoints + for w in activity.trackpoints: + self.add_trackpoint(track, w) + else: + for w in activity.trackpoints: + self.add_trackpoint(elem, w) def add_activity(self, element, activity): sport = activity.sport diff --git a/requirements.txt b/requirements.txt index e7d6104..7e6c2ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ distribute==0.6.34 -lxml==3.2.1 +lxml requests==1.2.3 wsgiref==0.1.2 From 31144b502f2572cd500f671ef5768d5a51bc3f3e Mon Sep 17 00:00:00 2001 From: CountablyInfinite Date: Wed, 18 Nov 2020 12:29:23 +0100 Subject: [PATCH 2/5] fixed some typos --- README.md | 2 +- export.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dda1d0c..53319fa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ python export.py Endomondo: Export the most recent n workouts (or ALL) as TCX files. Email: mymail@yourdomain.com Password: -Maximum number of workouts n (press Enter to ignore): 5 +Maximum number of workouts n (press Enter download ALL OF THEM): 5 Format for Garmin Connect? (press Enter to ignore): Y Files will be formatted to be compatible with Garmin Connects import function. fetched latest 5 workouts diff --git a/export.py b/export.py index 52bf34e..967ae08 100644 --- a/export.py +++ b/export.py @@ -50,7 +50,7 @@ def main(): print "Endomondo: export most recent n workouts (or ALL) as TCX files." mail = raw_input("Email: ") password = getpass.getpass() - maximum_workouts = raw_input("Maximum number of workouts n (press Enter to ignore): ") + maximum_workouts = raw_input("Maximum number of workouts n (press Enter to download ALL OF THEM): ") garmin = raw_input("Format for Garmin Connect? (press Enter to ignore): ") endomondo = Endomondo(mail, password, garmin) if garmin: From d1e7e6151713668e595f50bccfb3ad43f1675776 Mon Sep 17 00:00:00 2001 From: CountablyInfinite Date: Wed, 18 Nov 2020 13:34:44 +0100 Subject: [PATCH 3/5] Track element (wrapping Trackpoints) will now be used with all export, not only if Garmin Connect formatting is enabled --- export.py | 8 ++++---- lib/tcx.py | 15 +++------------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/export.py b/export.py index 967ae08..472e9f0 100644 --- a/export.py +++ b/export.py @@ -30,7 +30,7 @@ def create_directory(directory): # create the TCX file for the specified workout -def create_tcx_file(workout, garmin): +def create_tcx_file(workout): directory_name = 'export' activity = workout.get_activity() name = create_filename(workout) @@ -38,7 +38,7 @@ def create_tcx_file(workout, garmin): filename = os.path.join(directory_name, name) print "writing %s, %s, %s trackpoints" % (filename, activity.sport, len(activity.trackpoints)) - writer = tcx.Writer(garmin) + writer = tcx.Writer() tcxfile = writer.write(activity) if tcxfile: with open(filename, 'w') as f: @@ -67,14 +67,14 @@ def main(): print "Chunk after:" +str(after) workouts = endomondo.get_workouts(maximum_workouts, before, after) for workout in workouts: - create_tcx_file(workout,garmin) + create_tcx_file(workout) print "done." return 0 workouts = endomondo.get_workouts(maximum_workouts, before=None, after=None) print "fetched latest", len(workouts), "workouts" for workout in workouts: - create_tcx_file(workout, garmin) + create_tcx_file(workout) print "done." return 0 diff --git a/lib/tcx.py b/lib/tcx.py index 338efba..b59e8ab 100644 --- a/lib/tcx.py +++ b/lib/tcx.py @@ -40,9 +40,6 @@ def __init__(self): class Writer: - def __init__(self, garmin): - self.garmin = garmin - TCD_NAMESPACE = "http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2" XML_SCHEMA_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" @@ -106,15 +103,9 @@ def add_lap(self, element, activity, lap): self.add_property(elem, "Intensity", lap.intensity) self.add_property(elem, "Cadence", lap.cadence) self.add_property(elem, "TriggerMethod", lap.trigger_method) - # Garmin needs an extra Track wrapper - if self.garmin: - track = self.create_element(elem, "Track") - # Add trackpoints - for w in activity.trackpoints: - self.add_trackpoint(track, w) - else: - for w in activity.trackpoints: - self.add_trackpoint(elem, w) + track = self.create_element(elem, "Track") + for w in activity.trackpoints: + self.add_trackpoint(track, w) def add_activity(self, element, activity): sport = activity.sport From a8ac13d4083b264fe964f13653563b76c9de2a34 Mon Sep 17 00:00:00 2001 From: CountablyInfinite Date: Wed, 18 Nov 2020 13:44:44 +0100 Subject: [PATCH 4/5] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 53319fa..32c6a17 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Endomondo Export -This is an extended fork of https://github.com/yannickcarer/endomondo-export. Since Endomondo is shutting down it's service with the end of 2020 (what a year), i was searching for a way to export all my workouts (>1000) to Garmin Connect. yannickcarers script works fine (despite beeing python 2.x), but you can not export more than ~500 workouts since the server will timeout roughly around 500 exports. I added a feature than downloads the last 15 years as chunks of 500 trainings and also added the option to format the output for Garmin Connects import function, which needs an extra Wrapper around trackpoints and only allows "Running", "Biking" and "Other" as activity names. +This is an extended fork of https://github.com/yannickcarer/endomondo-export. Since Endomondo is shutting down it's service with the end of 2020 (what a year), i was searching for a way to export all my workouts (>1000) to Garmin Connect. yannickcarers script works fine (despite beeing python 2.x), but you can not export more than ~500 workouts since the server will timeout roughly around 500 exports. I added a feature than downloads the last 20 years as chunks of 500 trainings and also added the option to format the output for Garmin Connects import function, which needs an extra Wrapper around trackpoints and only allows "Running", "Biking" and "Other" as activity names. ## Usage -The script export.py may be used to backup the complete workout history. You will be asked for your Endomondo email and password, followed by the number of workouts you want to export (leave empty to export the last 15 years) and if you want to format the export to be compatible with Garmin Connect. +The script export.py may be used to backup the complete workout history. You will be asked for your Endomondo email and password, followed by the number of workouts you want to export (leave empty to export the last 20 years) and if you want to format the export to be compatible with Garmin Connect. **Important: If you have done more than 500 workouts in a year (congratulations!) you will run into problems using my script!** From 266e690fa4eb79ad4efa90dd15c23d64e0c7d352 Mon Sep 17 00:00:00 2001 From: CountablyInfinite Date: Fri, 9 Jul 2021 17:36:43 +0200 Subject: [PATCH 5/5] typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32c6a17..a2f4d0b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Endomondo Export -This is an extended fork of https://github.com/yannickcarer/endomondo-export. Since Endomondo is shutting down it's service with the end of 2020 (what a year), i was searching for a way to export all my workouts (>1000) to Garmin Connect. yannickcarers script works fine (despite beeing python 2.x), but you can not export more than ~500 workouts since the server will timeout roughly around 500 exports. I added a feature than downloads the last 20 years as chunks of 500 trainings and also added the option to format the output for Garmin Connects import function, which needs an extra Wrapper around trackpoints and only allows "Running", "Biking" and "Other" as activity names. +This is an extended fork of https://github.com/yannickcarer/endomondo-export. Since Endomondo is shutting down it's service with the end of 2020 (what a year), i was searching for a way to export all my workouts (>1000) to Garmin Connect. yannickcarers script works fine, but you can not export more than ~500 workouts since the server will timeout roughly around 500 exports. I added a feature that downloads the last 20 years as chunks of 500 trainings and also added the option to format the output for Garmin Connect, which needs an extra Wrapper around trackpoints and only allows "Running", "Biking" and "Other" as activity names. ## Usage