API tools faq
paste
Advertisement
Anordiel

GCExport

Anordiel
Sep 15th, 2017
178
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 17.95 KB | None | 0 0
raw download clone embed print report
  1. #!/usr/bin/python
  2.  
  3. """
  4. File: gcexport.py
  5. Author: Kyle Krafka (https://github.com/kjkjava/)
  6. Date: April 28, 2015
  7.  
  8. Description: Use this script to export your fitness data from Garmin Connect.
  9. See README.md for more information.
  10. """
  11.  
  12. from urllib import urlencode
  13. from datetime import datetime
  14. from getpass import getpass
  15. from sys import argv
  16. from os.path import isdir
  17. from os.path import isfile
  18. from os import mkdir
  19. from os import remove
  20. from xml.dom.minidom import parseString
  21.  
  22. import urllib2, cookielib, json
  23. from fileinput import filename
  24.  
  25. import argparse
  26. import zipfile
  27.  
  28. script_version = '1.0.0'
  29. current_date = datetime.now().strftime('%Y-%m-%d')
  30. activities_directory = './' + current_date + '_garmin_connect_export'
  31.  
  32. parser = argparse.ArgumentParser()
  33.  
  34. # TODO: Implement verbose and/or quiet options.
  35. # parser.add_argument('-v', '--verbose', help="increase output verbosity", action="store_true")
  36. parser.add_argument('--version', help="print version and exit", action="store_true")
  37. parser.add_argument('--username', help="your Garmin Connect username (otherwise, you will be prompted)", nargs='?')
  38. parser.add_argument('--password', help="your Garmin Connect password (otherwise, you will be prompted)", nargs='?')
  39.  
  40. parser.add_argument('-c', '--count', nargs='?', default="1",
  41. help="number of recent activities to download, or 'all' (default: 1)")
  42.  
  43. parser.add_argument('-f', '--format', nargs='?', choices=['gpx', 'tcx', 'original'], default="gpx",
  44. help="export format; can be 'gpx', 'tcx', or 'original' (default: 'gpx')")
  45.  
  46. parser.add_argument('-d', '--directory', nargs='?', default=activities_directory,
  47. help="the directory to export to (default: './YYYY-MM-DD_garmin_connect_export')")
  48.  
  49. parser.add_argument('-u', '--unzip',
  50. help="if downloading ZIP files (format: 'original'), unzip the file and removes the ZIP file",
  51. action="store_true")
  52.  
  53. args = parser.parse_args()
  54.  
  55. if args.version:
  56. print argv[0] + ", version " + script_version
  57. exit(0)
  58.  
  59. cookie_jar = cookielib.CookieJar()
  60. opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))
  61.  
  62. # url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers.
  63. def http_req(url, post=None, headers={}):
  64. request = urllib2.Request(url)
  65. request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1337 Safari/537.36') # Tell Garmin we're some supported browser.
  66. for header_key, header_value in headers.iteritems():
  67. request.add_header(header_key, header_value)
  68. if post:
  69. post = urlencode(post) # Convert dictionary to POST parameter string.
  70. response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError.
  71.  
  72. # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below.
  73. if response.getcode() != 200:
  74. raise Exception('Bad return code (' + response.getcode() + ') for: ' + url)
  75.  
  76. return response.read()
  77.  
  78. print 'Welcome to Garmin Connect Exporter!'
  79.  
  80. # Create directory for data files.
  81. if isdir(args.directory):
  82. print 'Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.'
  83.  
  84. username = args.username if args.username else raw_input('Username: ')
  85. password = args.password if args.password else getpass()
  86.  
  87. # Maximum number of activities you can request at once. Set and enforced by Garmin.
  88. limit_maximum = 100
  89.  
  90. # URLs for various services.
  91. url_gc_login = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false'
  92. url_gc_post_auth = 'https://connect.garmin.com/post-auth/login?'
  93. url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?'
  94. url_gc_gpx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/'
  95. url_gc_tcx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/'
  96. url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/'
  97.  
  98. # Initially, we need to get a valid session cookie, so we pull the login page.
  99. http_req(url_gc_login)
  100.  
  101. # Now we'll actually login.
  102. post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login.
  103. http_req(url_gc_login, post_data)
  104.  
  105. # Get the key.
  106. # TODO: Can we do this without iterating?
  107. login_ticket = None
  108. for cookie in cookie_jar:
  109. if cookie.name == 'CASTGC':
  110. login_ticket = cookie.value
  111. break
  112.  
  113. if not login_ticket:
  114. raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?')
  115.  
  116. # Chop of 'TGT-' off the beginning, prepend 'ST-0'.
  117. login_ticket = 'ST-0' + login_ticket[4:]
  118.  
  119. http_req(url_gc_post_auth + 'ticket=' + login_ticket)
  120.  
  121. # We should be logged in now.
  122. if not isdir(args.directory):
  123. mkdir(args.directory)
  124.  
  125. csv_filename = args.directory + '/activities.csv'
  126. csv_existed = isfile(csv_filename)
  127.  
  128. csv_file = open(csv_filename, 'a')
  129.  
  130. # Write header to CSV file
  131. if not csv_existed:
  132. csv_file.write('Activity ID,Activity Name,Description,Begin Timestamp,Begin Timestamp (Raw Milliseconds),End Timestamp,End Timestamp (Raw Milliseconds),Device,Activity Parent,Activity Type,Event Type,Activity Time Zone,Max. Elevation,Max. Elevation (Raw),Begin Latitude (Decimal Degrees Raw),Begin Longitude (Decimal Degrees Raw),End Latitude (Decimal Degrees Raw),End Longitude (Decimal Degrees Raw),Average Moving Speed,Average Moving Speed (Raw),Max. Heart Rate (bpm),Average Heart Rate (bpm),Max. Speed,Max. Speed (Raw),Calories,Calories (Raw),Duration (h:m:s),Duration (Raw Seconds),Moving Duration (h:m:s),Moving Duration (Raw Seconds),Average Speed,Average Speed (Raw),Distance,Distance (Raw),Max. Heart Rate (bpm),Min. Elevation,Min. Elevation (Raw),Elevation Gain,Elevation Gain (Raw),Elevation Loss,Elevation Loss (Raw)\n')
  133.  
  134. download_all = False
  135. if args.count == 'all':
  136. # If the user wants to download all activities, first download one,
  137. # then the result of that request will tell us how many are available
  138. # so we will modify the variables then.
  139. total_to_download = 1
  140. download_all = True
  141. else:
  142. total_to_download = int(args.count)
  143. total_downloaded = 0
  144.  
  145. # This while loop will download data from the server in multiple chunks, if necessary.
  146. while total_downloaded < total_to_download:
  147. # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100.
  148. if total_to_download - total_downloaded > 100:
  149. num_to_download = 100
  150. else:
  151. num_to_download = total_to_download - total_downloaded
  152.  
  153. search_params = {'start': total_downloaded, 'limit': num_to_download}
  154. # Query Garmin Connect
  155. result = http_req(url_gc_search + urlencode(search_params))
  156. json_results = json.loads(result) # TODO: Catch possible exceptions here.
  157.  
  158.  
  159. search = json_results['results']['search']
  160.  
  161. if download_all:
  162. # Modify total_to_download based on how many activities the server reports.
  163. total_to_download = int(search['totalFound'])
  164. # Do it only once.
  165. download_all = False
  166.  
  167. # Pull out just the list of activities.
  168. activities = json_results['results']['activities']
  169.  
  170. # Process each activity.
  171. for a in activities:
  172. # Display which entry we're working on.
  173. print a['activity']['activityId'],
  174. print '\t' + a['activity']['beginTimestamp']['display'] + ',',
  175. if args.format == 'gpx':
  176. data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.gpx'
  177. download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true'
  178. file_mode = 'w'
  179. elif args.format == 'tcx':
  180. data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.tcx'
  181. download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true'
  182. file_mode = 'w'
  183. elif args.format == 'original':
  184. data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.zip'
  185. fit_filename = args.directory + '/' + a['activity']['activityId'] + '.fit'
  186. download_url = url_gc_original_activity + a['activity']['activityId']
  187. file_mode = 'wb'
  188. else:
  189. raise Exception('Unrecognized format.')
  190.  
  191. if isfile(data_filename):
  192. print '\tData file already exists; skipping...'
  193. continue
  194. if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists.
  195. print '\tFIT data file already exists; skipping...'
  196. continue
  197.  
  198. # Download the data file from Garmin Connect.
  199. # If the download fails (e.g., due to timeout), this script will die, but nothing
  200. # will have been written to disk about this activity, so just running it again
  201. # should pick up where it left off.
  202. print '\tDL',
  203.  
  204. try:
  205. data = http_req(download_url)
  206. except urllib2.HTTPError as e:
  207. # Handle expected (though unfortunate) error codes; die on unexpected ones.
  208. if e.code == 500 and args.format == 'tcx':
  209. # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload.
  210. # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks.
  211. # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file,
  212. # as I believe Garmin provides a GPX file for every activity.
  213. print 'WRITE Empty TCX',
  214. data = ''
  215. elif e.code == 404 and args.format == 'original':
  216. # For manual activities (i.e., entered in online without a file upload), there is no original file.
  217. # Write an empty file to prevent redownloading it.
  218. print 'WRITE Empty FIT',
  219. data = ''
  220. else:
  221. raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').')
  222.  
  223. save_file = open(data_filename, file_mode)
  224. save_file.write(data)
  225. save_file.close()
  226.  
  227. # Write stats to CSV.
  228. empty_record = '"",'
  229.  
  230. csv_record = ''
  231.  
  232. csv_record += empty_record if 'activityId' not in a['activity'] else '"' + a['activity']['activityId'].replace('"', '""') + '",'
  233. csv_record += empty_record if 'activityName' not in a['activity'] else '"' + a['activity']['activityName']['value'].replace('"', '""') + '",'
  234. csv_record += empty_record if 'activityDescription' not in a['activity'] else '"' + a['activity']['activityDescription']['value'].replace('"', '""') + '",'
  235. csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['display'].replace('"', '""') + '",'
  236. csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['millis'].replace('"', '""') + '",'
  237. csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['display'].replace('"', '""') + '",'
  238. csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['millis'].replace('"', '""') + '",'
  239. csv_record += empty_record if 'device' not in a['activity'] else '"' + a['activity']['device']['display'].replace('"', '""') + ' ' + a['activity']['device']['version'].replace('"', '""') + '",'
  240. csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['parent']['display'].replace('"', '""') + '",'
  241. csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['display'].replace('"', '""') + '",'
  242. csv_record += empty_record if 'eventType' not in a['activity'] else '"' + a['activity']['eventType']['display'].replace('"', '""') + '",'
  243. csv_record += empty_record if 'activityTimeZone' not in a['activity'] else '"' + a['activity']['activityTimeZone']['display'].replace('"', '""') + '",'
  244. csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",'
  245. csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",'
  246. csv_record += empty_record if 'beginLatitude' not in a['activity'] else '"' + a['activity']['beginLatitude']['value'].replace('"', '""') + '",'
  247. csv_record += empty_record if 'beginLongitude' not in a['activity'] else '"' + a['activity']['beginLongitude']['value'].replace('"', '""') + '",'
  248. csv_record += empty_record if 'endLatitude' not in a['activity'] else '"' + a['activity']['endLatitude']['value'].replace('"', '""') + '",'
  249. csv_record += empty_record if 'endLongitude' not in a['activity'] else '"' + a['activity']['endLongitude']['value'].replace('"', '""') + '",'
  250. csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile"
  251. csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['value'].replace('"', '""') + '",'
  252. csv_record += empty_record if 'maxHeartRate' not in a['activity'] else '"' + a['activity']['maxHeartRate']['display'].replace('"', '""') + '",'
  253. csv_record += empty_record if 'weightedMeanHeartRate' not in a['activity'] else '"' + a['activity']['weightedMeanHeartRate']['display'].replace('"', '""') + '",'
  254. csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile"
  255. csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['value'].replace('"', '""') + '",'
  256. csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['display'].replace('"', '""') + '",'
  257. csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['value'].replace('"', '""') + '",'
  258. csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['display'].replace('"', '""') + '",'
  259. csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['value'].replace('"', '""') + '",'
  260. csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['display'].replace('"', '""') + '",'
  261. csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['value'].replace('"', '""') + '",'
  262. csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",'
  263. csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['value'].replace('"', '""') + '",'
  264. csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['withUnit'].replace('"', '""') + '",'
  265. csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['value'].replace('"', '""') + '",'
  266. csv_record += empty_record if 'minHeartRate' not in a['activity'] else '"' + a['activity']['minHeartRate']['display'].replace('"', '""') + '",'
  267. csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",'
  268. csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",'
  269. csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['withUnit'].replace('"', '""') + '",'
  270. csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['value'].replace('"', '""') + '",'
  271. csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['withUnit'].replace('"', '""') + '",'
  272. csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['value'].replace('"', '""') + '"'
  273. csv_record += '\n'
  274.  
  275. csv_file.write(csv_record.encode('utf8'))
  276.  
  277. if args.format == 'gpx':
  278. # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill),
  279. # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data.
  280. # N.B. You can omit the XML parse (and the associated log messages) to speed things up.
  281. gpx = parseString(data)
  282. gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0
  283.  
  284. if gpx_data_exists:
  285. print 'Done. GPX data saved.'
  286. else:
  287. print 'Done. No track points found.'
  288. elif args.format == 'original':
  289. if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension.
  290. print "UNZIP...",
  291. zip_file = open(data_filename, 'rb')
  292. z = zipfile.ZipFile(zip_file)
  293. for name in z.namelist():
  294. z.extract(name, args.directory)
  295. zip_file.close()
  296. remove(data_filename)
  297. print 'Done.'
  298. else:
  299. # TODO: Consider validating other formats.
  300. print 'Done.'
  301. total_downloaded += num_to_download
  302. # End while loop for multiple chunks.
  303.  
  304. csv_file.close()
  305.  
  306. print 'Done!'
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!