source: modules/mod_mapData.py @ ec56a8a

Last change on this file since ec56a8a was ec56a8a, checked in by xkolman2 <xkolman2@…>, 4 years ago

replace active tracklog index by active tracklog path variable
-> fix the mess resulting from inconsistent indexes
make all modules load the tracklogs throught a method, not directly
-> safer, more transparent

git-svn-id: https://nlp.fi.muni.cz/svn/gps_navigace/trunk@156 0858a4d0-ffff-46e5-938e-62b5ecb34222

  • Property mode set to 100644
File size: 35.9 KB
Line 
1#!/usr/bin/python
2#----------------------------------------------------------------------------
3# Handles downloading of map data
4#----------------------------------------------------------------------------
5# Copyright 2008, Oliver White
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19#---------------------------------------------------------------------------
20from base_module import ranaModule
21from time import sleep
22from tilenames import *
23from time import clock
24import time
25import os
26import statvfs
27import sys
28import geo
29import string
30import urllib, urllib2, threading
31from threadpool import threadpool
32from threading import Thread
33
34import socket
35timeout = 30 # this sets timeout for all sockets
36socket.setdefaulttimeout(timeout)
37
38
39
40#from modules.pyrender.tilenames import xy2latlon
41import sys
42if(__name__ == '__main__'):
43  sys.path.append('pyroutelib2')
44else:
45  sys.path.append('modules/pyroutelib2')
46
47import tiledata
48
49def getModule(m,d):
50  return(mapData(m,d))
51
52class mapData(ranaModule):
53  """Handle downloading of map data"""
54 
55  def __init__(self, m, d):
56    ranaModule.__init__(self, m, d)
57    self.stopThreading = True
58    self.currentDownloadList = [] # list of files and urls for the current download batch
59#    self.currentTilesToGet = [] # used for reporting the (actual) size of the tiles for download
60    self.sizeThread = None
61    self.getFilesThread = None
62    self.aliasForSet = self.set
63    self.lastMenuRedraw = 0
64    self.notificateOnce = True
65    self.scroll = 0
66
67  def listTiles(self, route):
68    """List all tiles touched by a polyline"""
69    tiles = {}
70    for pos in route:
71      (lat,lon) = pos
72      (tx,ty) = tileXY(lat, lon, 15)
73      tile = "%d,%d" % (tx,ty)
74      if(not tiles.has_key(tile)):
75        tiles[tile] = True
76    return(tiles.keys())
77
78  def checkTiles(self, tilesToDownload):
79    """
80    Get tiles that need to be downloaded and look if we dont already have some of these tiles,
81    then generate a set of ('url','filename') touples and send them to the threaded downloader
82    """
83    print "Checking if there are duplicated tiles"
84    start = clock()
85#    self.currentTilesToGet = tilesToDownload # this is for displaying the tiles for debugging reasons
86    tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles
87    layer = self.get('layer', None) # TODO: manual layer setting
88    maplayers = self.get('maplayers', None) # a distionary describing supported maplayers
89    extension = maplayers[layer]['type'] # what is the extension for the current layer ?
90    folderPrefix = maplayers[layer]['folderPrefix'] # what is the extension for the current layer ?
91#    alreadyDownloadedTiles = set(os.listdir(tileFolderPath)) # already dowloaded tiles
92
93    mapTiles = self.m.get('mapTiles', None)
94
95    neededTiles = []
96
97    for tile in tilesToDownload: # check what tiles are already stored
98      (z,x,y) = (tile[2],tile[0],tile[1])
99      filePath = tileFolder + mapTiles.imagePath(x, y, z, folderPrefix, extension)
100      if not os.path.exists(filePath): # we dont have this file
101        neededTiles.append(tile)
102#      if not os.path.exists(filePath): # we dont have this file
103#        url = self.getTileUrl(x, y, z, layer) # generate url
104#        urlsAndFilenames.append((url,filePath)) # store url and path
105
106    print "Downloading %d new tiles." % len(neededTiles)
107    print "Removing already available tiles from dl took %1.2f ms" % (1000 * (clock() - start))
108    return neededTiles
109
110
111  def getTileUrlAndPath(self,x,y,z,layer):
112    mapTiles = self.m.get('mapTiles', None)
113    tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles
114    maplayers = self.get('maplayers', None) # a distionary describing supported maplayers
115    extension = maplayers[layer]['type'] # what is the extension for the current layer ?
116    folderPrefix = maplayers[layer]['folderPrefix'] # what is the extension for the current layer ?
117    url = self.getTileUrl(x, y, z, layer) # generate url
118    filePath = tileFolder + mapTiles.imagePath(x, y, z, folderPrefix, extension)
119    fileFolder = tileFolder + mapTiles.imageFolder(x, z, folderPrefix)
120    return (url,filePath, fileFolder)
121
122  def addToQueue(self, neededTiles):
123    """load urls and filenames to download queue,
124       optionaly check for duplicates
125    """
126
127    tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles
128    print "tiles for the batch will be downloaded to: %s" % tileFolder
129
130    check = self.get('checkTiles', False)
131    if check:
132      neededTiles = self.checkTiles(neededTiles)
133
134    self.currentDownloadList = neededTiles # load the files to the download queue variable
135
136  def getTileUrl(self,x,y,z,layer):
137      """Return url for given tile coorindates and layer"""
138      mapTiles = self.m.get('mapTiles', None)
139      url = mapTiles.getTileUrl(x,y,z,layer)
140      return url
141
142# adapted from: http://www.artfulcode.net/articles/multi-threading-python/
143  class FileGetter(threading.Thread):
144    def __init__(self, url, filename):
145        self.url = url
146        self.filename = filename
147#        self.result = None
148        threading.Thread.__init__(self)
149
150    def getResult(self):
151        return self.result
152
153    def run(self):
154        try:
155            url = self.url
156            filename = self.filename
157            urllib.urlretrieve(url, filename)
158            self.result = filename
159            print "download of %s finished" % filename
160        except IOError:
161            print "Could not open document: %s" % url
162
163  def handleMessage(self, message):
164    if(message == "refreshTilecount"):
165      size = int(self.get("downloadSize", 4))
166      type = self.get("downloadType")
167      if(type != "data"):
168        print "Error: mod_mapData can't download %s" % type
169        return
170
171      # update the info when refreshing tilecount and and no dl/size estimation is active
172
173      if self.sizeThread:
174        if self.sizeThread.isAlive() == False:
175          self.sizeThread = None
176
177      if self.getFilesThread:
178        if self.getFilesThread.isAlive() == False:
179          self.getFilesThread = None
180     
181      location = self.get("downloadArea", "here") # here or route
182
183      z = self.get('z', 15) # this is the currewnt zoomlevel as show on the map screen
184      minZ = z - int(self.get('zoomUpSize', 0)) # how many zoomlevels up (from current zoomelevel) should we download ?
185      if minZ < 0:
186        minZ = 0
187      maxZ = z + int(self.get('zoomDownSize', 0)) # how many zoomlevels down (from current zoomlevel) should we download ?
188
189      layer = self.get('layer', None)
190      maplayers = self.get('maplayers', None)
191      if maplayers == None:
192        maxZoomLimit == 17
193      else:
194        maxZoomLimit = maplayers[layer]['maxZoom']
195
196      if maxZ > maxZoomLimit:
197        maxZ = 17 #TODO: make layer specific
198#      z = currentZ # current Zoomlevel
199      diffZ = maxZ - minZ
200      midZ = int(minZ + (diffZ/2.0))
201
202      """well, its not exactly middle, its jut a value that decides, if we split down or just round up
203         splitting from a zoomlevel too high can lead to much more tiles than requested
204         for example, we want tiles for a 10 km radius but we choose to split from a zoomlevel, where a tile is
205         20km*20km and our radius intersects four of these tiles, when we split these tiles, we get tiles for an
206         are of 40km*40km, instead of the requested 10km
207         therefore, zoom level 15 is used as the minimum number for splitting tiles down
208         when the maximum zoomlevel from the range requested is less than 15, we dont split at all"""
209      if midZ < 15 and maxZ < 15:
210        midZ = maxZ
211      else:
212        midZ = 15
213      print "max: %d, min: %d, diff: %d, middle:%d" % (maxZ, minZ, diffZ, midZ)
214
215      if(location == "here"): 
216        # Find which tile we're on
217        pos = self.get("pos",None)
218        if(pos != None):
219          (lat,lon) = pos
220          # be advised: the xy in this case are not screen coordinates but tile coordinates
221          (x,y) = latlon2xy(lat,lon,midZ)
222          tilesAroundHere = set(self.spiral(x,y,midZ,size)) # get tiles around our position as a set
223          # now get the tiles from other zoomlevels as specified
224          zoomlevelExtendedTiles = self.addOtherZoomlevels(tilesAroundHere, midZ, maxZ, minZ)
225
226          self.addToQueue(zoomlevelExtendedTiles) # load the files to the download queue
227
228
229      if(location == "route"):
230        loadTl = self.m.get('loadTracklogs', None) # get the tracklog module
231        GPXTracklog = loadTl.getActiveTracklog()
232        """because we dont need all the information in the original list and
233        also might need to add interpolated points, we make a local copy of
234        the original list"""
235        #latLonOnly = filter(lambda x: [x.latitude,x.longitude])
236        trackpointsListCopy = map(lambda x: {'latitude': x.latitude,'longitude': x.longitude}, GPXTracklog.trackpointsList[0])[:]
237        tilesToDownload = self.getTilesForRoute(trackpointsListCopy, size, midZ)
238        zoomlevelExtendedTiles = self.addOtherZoomlevels(tilesToDownload, midZ, maxZ, minZ)
239
240        self.addToQueue(zoomlevelExtendedTiles) # load the files to the download queue
241       
242
243      if(location == "view"):
244        proj = self.m.get('projection', None)
245        (screenCenterX,screenCenterY) = proj.screenPos(0.5, 0.5) # get pixel coordinates for the screen center
246        (lat,lon) = proj.xy2ll(screenCenterX,screenCenterY) # convert to geographic coordinates
247        (x,y) = latlon2xy(lat,lon,midZ) # convert to tile coordinates
248        tilesAroundView = set(self.spiral(x,y,midZ,size)) # get tiles around these coordinates
249        # now get the tiles from other zoomlevels as specified
250        zoomlevelExtendedTiles = self.addOtherZoomlevels(tilesAroundView, midZ, maxZ, minZ)
251
252        self.addToQueue(zoomlevelExtendedTiles) # load the files to the download queue
253
254    if(message == "getSize"):
255      """will now ask the server and find the combined size if tiles in the batch"""
256      self.set("sizeStatus", 'unknown') # first we set the size as unknown
257      neededTiles = self.currentDownloadList
258      layer = self.get('layer', None)
259      print "getting size"
260      if len(neededTiles) == 0:
261        print "cant get combined size, the list is empty"
262        return
263
264      if self.sizeThread != None:
265        if self.sizeThread.finished == False:
266          print "size check already in progress"
267          return
268
269      self.totalSize = 0
270      maxThreads = self.get('maxSizeThreads', 5)
271      sizeThread = self.GetSize(neededTiles, layer, self.getTileUrlAndPath, maxThreads) # the seccond parameter is the max number of threads TODO: tweak this
272      print  "getSize received, starting sizeThread"
273      sizeThread.start()
274      self.sizeThread = sizeThread
275
276    if(message == "download"):
277      """get tilelist and download the tiles using threads"""
278      neededTiles = self.currentDownloadList
279      layer = self.get('layer', None)
280      print "starting download"
281      if len(neededTiles) == 0:
282        print "cant download an empty list"
283        return
284
285      if self.getFilesThread != None:
286        if self.getFilesThread.finished == False:
287          print "download already in progress"
288          return
289       
290      maxThreads = self.get('maxDlThreads', 5)
291      getFilesThread = self.GetFiles(neededTiles, layer, self.getTileUrlAndPath, maxThreads)
292      getFilesThread.start()
293      self.getFilesThread = getFilesThread
294
295    if(message == "up"):
296      if(self.scroll > 0):
297        self.scroll -= 1
298        self.set('needRedraw', True)
299    if(message == "down"):
300      print "down"
301      self.scroll += 1
302      self.set('needRedraw', True)
303    if(message == "reset"):
304      self.scroll = 0
305      self.set("needRedraw", True)
306
307
308  def addOtherZoomlevels(self, tiles, tilesZ, maxZ, minZ):
309    """expand the tile coverage to other zoomlevels
310    maxZ = maximum NUMERICAL zoom, 17 for eaxmple
311    minZ = minimum NUMERICAL zoom, 0 for example
312    we use two different methods to get the needed tiles:
313    * spliting the tiles from one zoomlevel down to the other
314    * rounding the tiles cooridnates to get tiles from one zoomlevel up
315    we choose a zoomlevel (tilesZ) and then split down and round down from it
316    * tilesZ is determined in the handle message method,
317    it is the zoomlevel on which we compute which tiles are in our download radius
318    -> if tilesZ is too low, this initial tile finding can take too long
319    -> if tilesZ is too high, the tiles could be much larger than our dl. radius and we would
320    be downloading much more tiles than needed
321    => for now, if we get tilesZ (called midZ in handle message) that is lower than 15,
322    we set it to the lowest zoomlevel, so we get dont get too much unneeded tiles when splitting
323    """
324    start = clock()
325    extendedTiles = tiles.copy()
326
327
328
329    """start of the tile splitting code"""
330    previousZoomlevelTiles = None # we will splitt the tiles from the previous zoomlevel
331    print "splitting down"
332    for z in range(tilesZ, maxZ): # to max zoom (fo each z we split one zoomlevel down)
333      newTilesFromSplit = set() # tiles from the splitting go there
334      if previousZoomlevelTiles == None: # this is the first iteration
335        previousZoomlevelTiles = tiles.copy()
336      for tile in previousZoomlevelTiles:
337        x = tile[0]
338        y = tile[1]
339        """
340        now we split each tile to 4 tiles on a higher zoomlevel nr
341        see: http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Subtiles
342        for a tile with cooridnates x,y:
343        2x,2y  |2x+1,2y
344        2x,2y+1|2x+1,2y+1
345        """
346        leftUpperTile = (2*x, 2*y, z+1)
347        rightUpperTile = (2*x+1, 2*y, z+1)
348        leftLowerTile = (2*x, 2*y+1, z+1)
349        rightLowerTile = (2*x+1, 2*y+1, z+1)
350        newTilesFromSplit.add(leftUpperTile)
351        newTilesFromSplit.add(rightUpperTile)
352        newTilesFromSplit.add(leftLowerTile)
353        newTilesFromSplit.add(rightLowerTile)
354      extendedTiles.update(newTilesFromSplit) # add the new tiles to the main set
355      print "we are at z=%d, %d new tiles from %d" % (z, len(newTilesFromSplit), (z+1))
356      previousZoomlevelTiles = newTilesFromSplit # set the new tiles s as prev. tiles for next iteration
357
358    """start of the tile cooridnates rounding code"""
359    previousZoomlevelTiles = None # we will the tile cooridnates to get tiles for the upper level
360    print "rounding up"
361    r = range(minZ, tilesZ) # we go from the tile-z up, e.g. a seqence of progresivly smaller integers
362    r.reverse()
363    for z in r:
364      newTilesFromRounding = set() # tiles from the rounding go there
365      if previousZoomlevelTiles == None: # this is the first iteration
366        previousZoomlevelTiles = tiles.copy()
367      for tile in previousZoomlevelTiles:
368        x = tile[0]
369        y = tile[1]
370        """as per: http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Subtiles
371        we divide each cooridnate with 2 to get the upper tile
372        some upper tiles can be found up to four times, so this could be most probably
373        optimized if need be (for charting the Jupiter, Sun or a Dyson sphere ? :)"""
374        upperTileX = int(x/2.0)
375        upperTileY = int(y/2.0)
376        upperTile = (upperTileX,upperTileY, z)
377        newTilesFromRounding.add(upperTile)
378      extendedTiles.update(newTilesFromRounding) # add the new tiles to the main set
379      print "we are at z=%d, %d new tiles" % (z, len(newTilesFromRounding))
380      previousZoomlevelTiles = newTilesFromRounding # set the new tiles s as prev. tiles for next iteration
381
382      print "nr of tiles after extend: %d" % len(extendedTiles)
383    print "Extend took %1.2f ms" % (1000 * (clock() - start))
384
385    del tiles
386
387    return extendedTiles
388
389  class GetSize(Thread):
390    """a class for getting size of files on and url list"""
391    def __init__(self, neededTiles, layer, namingFunction, maxThreads):
392      Thread.__init__(self)
393      self.neededTiles=neededTiles
394      self.maxThreads=maxThreads
395      self.layer=layer
396      self.namingFunction=namingFunction
397      self.processed = 0
398      self.urlCount = len(neededTiles)
399      self.totalSize = 0
400      self.finished = False
401      self.quit = False
402#      self.set("sizeStatus", 'inProgress') # the size is being processed
403
404    def getSizeForURL(self, tile):
405      """NOTE: getting size info for a sigle tile seems to take from 30 to 130ms on fast connection"""
406#      start = clock()
407      try:
408        (z,x,y) = (tile[2],tile[0],tile[1])
409        (url, filename, folder) = self.namingFunction(x, y, z, self.layer)
410        if not os.path.exists(filename):
411          url = urllib.urlopen(url) # open url and get mime info
412          urlInfo = url.info()
413          size = int(urlInfo['Content-Length']) # size in bytes
414          url.close()
415        else:
416          size = 0
417      except IOError:
418        print "Could not open document: %s" % url
419        size = 0 # the url errored out, so we just say it  has zero size
420#      print "Size lookup took %1.2f ms" % (1000 * (clock() - start))
421      return size
422
423    def processURLSize(self, request, result):
424      """process the ruseult from the getSize threads"""
425      self.processed = self.processed + 1
426#      print "**** Size from request #%s: %r b" % (request.requestID, result)
427      self.totalSize+=result
428#      print "**** Total size: %r B, %d/%d done" % (self.totalSize, self.processed, self.urlCount)
429
430    def run(self):
431      start = clock()
432      neededTiles=self.neededTiles
433      maxThreads=self.maxThreads
434      requests = threadpool.makeRequestsWithTuples(self.getSizeForURL, neededTiles, self.processURLSize)
435      mainPool = threadpool.ThreadPool(maxThreads)
436      print "GetSize: mainPool created"
437#      for req in requests:
438#          mainPool.putRequest(req)
439      map(lambda x: mainPool.putRequest(x) ,requests)
440      print "Added %d URLS to check for size." % self.urlCount
441      while True:
442          try:
443              time.sleep(0.5) # this governs how often we check status of the worker threads
444              mainPool.poll()
445              print "Main thread working...",
446              print "(active worker threads: %i)" % (threading.activeCount()-1, )
447              if self.quit == True:
448                print "get size quiting"
449                break
450          except threadpool.NoResultsPending:
451              print "**** No pending results."
452              print "Total size lookup took %1.2f ms" % (1000 * (clock() - start))
453              self.finished = True
454              mainPool.dismissWorkers(maxThreads)
455              break
456#      if mainPool.dismissedWorkers:
457#          print "Joining all dismissed worker threads..."
458#          mainPool.joinAllDismissedWorkers()
459
460
461  class GetFiles(Thread):
462    def __init__(self, neededTiles, layer, namingFunction, maxThreads):
463      Thread.__init__(self)
464      self.neededTiles = neededTiles
465      self.maxThreads = maxThreads
466      self.layer=layer
467      self.namingFunction=namingFunction
468      self.processed = 0
469      self.urlCount = len(neededTiles)
470      self.finished = False
471      self.quit = False
472
473    def saveTileForURL(self, tile):
474      (z,x,y) = (tile[2],tile[0],tile[1])
475      (url, filename, folder) = self.namingFunction(x, y, z, self.layer)
476
477      try:
478        if not os.path.exists(folder):
479          os.makedirs(folder)
480          urllib.urlretrieve(url,filename) # open url and seve it to file
481        else:
482          if not os.path.exists(filename):
483            urllib.urlretrieve(url,filename) # we dont download files we already have
484
485      except Exception, e:
486        print "Saving tile %s failed: %s" % (url, e)
487        return "nok"
488      return "ok"
489
490    def processSaveTile(self, request, result):
491      #TODO: redownload failed tiles
492      self.processed = self.processed + 1
493      self.tileThreadingStatus = self.processed
494#      print "**** Downloading: %d of %d tiles done. Status:%r" % (self.processed, self.urlCount, result)
495
496    def run(self):
497      neededTiles = self.neededTiles
498      maxThreads = self.maxThreads
499      requests = threadpool.makeRequestsWithTuples(self.saveTileForURL, neededTiles, self.processSaveTile)
500      mainPool = threadpool.ThreadPool(maxThreads)
501      print "GetFiles: mainPool created"
502#      for req in requests:
503#          mainPool.putRequest(req)
504      map(lambda x: mainPool.putRequest(x) ,requests)
505      print "Added %d URLS to check for size." % self.urlCount
506      while True:
507          try:
508              time.sleep(0.5)
509              print mainPool.poll()
510              print "Main thread working...",
511              print "(active worker threads: %i)" % (threading.activeCount()-1, )
512              if self.quit == True:
513                print "get size quiting"
514                break
515          except threadpool.NoResultsPending:
516              print "**** No pending results."
517              self.finished = True
518              mainPool.dismissWorkers(maxThreads)
519              break
520#      if mainPool.dismissedWorkers:
521#          print "Joining all dismissed worker threads..."
522#          mainPool.joinAllDismissedWorkers()
523
524#  def handleThreExc(self, request, exc_info):
525#        if not isinstance(exc_info, tuple):
526#            # Something is seriously wrong...
527#            print request
528#            print exc_info
529#            raise SystemExit
530#        print "**** Exception occured in request #%s: %s" % \
531#          (request.requestID, exc_info)
532
533  def expand(self, tileset, amount=1):
534    """Given a list of tiles, expand the coverage around those tiles"""
535    tiles = {}
536    tileset = [[int(b) for b in a.split(",")] for a in tileset]
537    for tile in tileset:
538      (x,y) = tile
539      for dx in range(-amount, amount+1):
540        for dy in range(-amount, amount+1):
541          tiles["%d,%d" % (x+dx,y+dy)] = True
542    return(tiles.keys())
543
544  def spiral(self, x, y, z, distance):
545    (x,y) = (int(round(x)),int(round(y)))
546    """for now we are downloading just tiles,
547    so I modified this to round the coordinates right after we get them"""
548    class spiraller:
549      def __init__(self,x,y,z):
550        self.x = x
551        self.y = y
552        self.z = z
553        self.tiles = [(x,y,z)]
554      def moveX(self,dx, direction):
555        for i in range(dx):
556          self.x += direction
557          self.touch(self.x, self.y, self.z)
558      def moveY(self,dy, direction):
559        for i in range(dy):
560          self.y += direction
561          self.touch(self.x, self.y, self.z)
562      def touch(self,x,y,z):
563        self.tiles.append((x,y,z))
564       
565    s =spiraller(x,y,z)
566    for d in range(1,distance+1):
567      s.moveX(1, 1) # 1 right
568      s.moveY(d*2-1, -1) # d*2-1 up
569      s.moveX(d*2, -1)   # d*2 left
570      s.moveY(d*2, 1)    # d*2 down
571      s.moveX(d*2, 1)    # d*2 right
572    return(s.tiles)
573
574  def update(self):
575#    """because it seems, that unless we force redraw the window
576#    the threads will be stuck, we poke them while they are running
577#    TODO: maybe the this could be done more elegantly ?"""
578#    if self.sizeThread != None and self.sizeThread.finished == False:
579#      self.set('needRedraw', True)
580#      print "refreshing the GetSize thread"
581#    if self.getFilesThread != None and self.getFilesThread.finished == False:
582#      self.set('needRedraw', True)
583##      print "refreshing the GetFiles thread"
584    pass
585
586  def getTilesForRoute(self, route, radius, z):
587    """get tilenamames for tiles around the route for given radius and zoom"""
588    """ now we look whats the distance between each two trackpoints,
589    if it is larger than the tracklog radius, we add aditioanl interpolated points,
590    so we get continuous coverage for the tracklog """
591    first = True
592    interpolatedPoints = []
593    for point in route:
594      if first: # the first point has no previous point
595        (lastLat, lastLon) = (point['latitude'], point['longitude'])
596        first = False
597        continue
598      (thisLat, thisLon) = (point['latitude'], point['longitude'])
599      distBtwPoints = geo.distance(lastLat, lastLon, thisLat, thisLon)
600      """if the distance between points was greater than the given radius for tiles,
601      there would be no continuous coverage for the route"""
602      if distBtwPoints > radius:
603        """so we call this recursive function to interpolate points between
604        points that are too far apart"""
605        interpolatedPoints.extend(self.addPointsToLine(lastLat, lastLon, thisLat, thisLon, radius))
606      (lastLat, lastLon) = (thisLat, thisLon)
607    """because we dont care about what order are the points in this case,
608    we just add the interpolated points to the end"""
609    route.extend(interpolatedPoints)
610    start = clock()
611    tilesToDownload = set()
612    for point in route: #now we iterate over all points of the route
613      (lat,lon) = (point['latitude'], point['longitude'])
614      # be advised: the xy in this case are not screen coordinates but tile coordinates
615      (x,y) = latlon2xy(lat,lon,z)
616      # the spiral gives us tiles around coordinates for a given radius
617      currentPointTiles = self.spiral(x,y,z,radius)
618      """now we take the resulting list  and process it in suach a way,
619      that the tiles coordinates can be stored in a set,
620      so we will save only unique tiles"""
621      outputSet = set(map(lambda x: tuple(x), currentPointTiles))
622      tilesToDownload.update(outputSet)
623    print "Listing tiles took %1.2f ms" % (1000 * (clock() - start))
624    print "unique tiles %d" % len(tilesToDownload)
625    return tilesToDownload
626
627  def addPointsToLine(self, lat1, lon1, lat2, lon2, maxDistance):
628    """experimental (recursive) function for adding aditional points between two coordinates,
629    until their distance is less or or equal to maxDistance
630    (this is actually a wrapper for a local recursive function)"""
631    pointsBetween = []
632    def localAddPointsToLine(lat1, lon1, lat2, lon2, maxDistance):
633      distance = geo.distance(lat1, lon1, lat2, lon2)
634      if distance <= maxDistance: # the termination criterium
635        return
636      else:
637        middleLat = (lat1 + lat2)/2.0 # fin the midpoint between the two points
638        middleLon = (lon1 + lon2)/2.0
639        pointsBetween.extend([{'latitude': middleLat,'longitude': middleLon}])
640        # process the 2 new line segments
641        localAddPointsToLine(lat1, lon1, middleLat, middleLon, maxDistance)
642        localAddPointsToLine(middleLat, middleLon, lat2, lon2, maxDistance)
643
644    localAddPointsToLine(lat1, lon1, lat2, lon2, maxDistance) # call the local function
645    return pointsBetween
646
647  def drawMenu(self, cr, menuName):
648    # is this menu the correct menu ?
649    if menuName == 'batchTileDl':
650      """in order for the threeds to work normally, it is needed to pause the main loop for a while
651      * this works only for this menu, in other menus (even the edit  menu) the threads will be slow to start
652      * when looking at map, the threads behave as expected :)
653      * so, when downloading:
654      -> look at the map OR the batch progress :)**"""
655      time.sleep(0.5)
656      self.set('needRedraw', True)
657      (x1,y1,w,h) = self.get('viewport', None)
658      self.set('dataMenu', 'edit')
659      menus = self.m.get("menu",None)
660      sizeThread = self.sizeThread
661      getFilesThread = self.getFilesThread
662      self.set("batchMenuEntered", True)
663
664      if w > h:
665        cols = 4
666        rows = 3
667      elif w < h:
668        cols = 3
669        rows = 4
670      elif w == h:
671        cols = 4
672        rows = 4
673
674      dx = w / cols
675      dy = h / rows
676      # * draw "escape" button
677      menus.drawButton(cr, x1, y1, dx, dy, "", "up", "menu:rebootDataMenu|set:menu:main")
678      # * draw "edit" button
679      menus.drawButton(cr, (x1+w)-2*dx, y1, dx, dy, "edit", "tools", "menu:setupEditBatchMenu|set:menu:editBatch")
680      # * draw "start" button
681      menus.drawButton(cr, (x1+w)-1*dx, y1, dx, dy, "start", "start", "mapData:download")
682      # * draw the combined info area and size button (aka "box")
683      boxX = x1
684      boxY = y1+dy
685      boxW = w
686      boxH = h-dy
687      menus.drawButton(cr, boxX, boxY, boxW, boxH, "", "3h", "mapData:getSize")
688
689      # * display information about download status
690      getFilesText = self.getFilesText(getFilesThread)
691      getFilesTextX = boxX + dx/8
692      getFilesTextY = boxY + boxH*1/4
693      self.showText(cr, getFilesText, getFilesTextX, getFilesTextY, w-dx/4, 40)
694
695      # * display information about size of the tiles
696      sizeText = self.getSizeText(sizeThread)
697      sizeTextX = boxX + dx/8
698      sizeTextY = boxY + boxH*2/4
699      self.showText(cr, sizeText, sizeTextX, sizeTextY, w-dx/4, 40)
700
701      # * display information about free space available (for the filesystem with the tilefolder)
702      freeSpaceText = self.getFreeSpaceText()
703      freeSpaceTextX = boxX + dx/8
704      freeSpaceTextY = boxY + boxH * 3/4
705      self.showText(cr, freeSpaceText, freeSpaceTextX, freeSpaceTextY, w-dx/4, 40)
706
707    if menuName == 'chooseRouteForDl':
708
709
710      menus = self.m.get('menu', None)
711      tracks = self.m.get('loadTracklogs', None).getTracklogList()
712      print tracks
713
714      def describeTracklog(index, category, tracks):
715        """describe a tracklog list item"""
716        track = tracks[index]
717
718        action = "set:activeTracklogPath:%s|loadTracklogs:loadActive" % track['path']
719
720        status = self.get('editBatchMenuActive', False)
721        if status == True:
722          action+= '|menu:setupEditBatchMenu|set:menu:editBatch'
723        else:
724          action+= '|set:menu:data2'
725        name = tracks[index]['filename']
726        desc = 'cat.: ' + track['cat'] + '   size:' + track['size'] + '   last modified:' + track['lastModified']
727#        if track.perElevList:
728#          length = track.perElevList[-1][0]
729#          units = self.m.get('units', None)
730#          desc+=", %s" % units.km2CurrentUnitString(length)
731        return (name,desc,action)
732
733
734      status = self.get('editBatchMenuActive', False)
735      if status == True:
736        parent = 'editBatch'
737      else:
738        parent = 'data'
739      scrollMenu = 'mapData'
740      menus.drawListableMenuControls(cr, menuName, parent, scrollMenu)
741      menus.drawListableMenuItems(cr, tracks, self.scroll, describeTracklog)
742
743
744
745  def getFilesText(self, getFilesThread):
746    """return a string describing status of the download threads"""
747    tileCount = len(self.currentDownloadList)
748    if tileCount == 0:
749      return "All tiles for this area are available."
750    elif getFilesThread == None:
751      return ( "Press Start to download ~ %d tiles." % tileCount)
752    elif getFilesThread.isAlive() == True:
753      totalTileCount = getFilesThread.urlCount
754      currentTileCount = getFilesThread.processed
755      text = "Downloading: %d of %d tiles complete" % (currentTileCount, totalTileCount)
756      self.notificateOnce = True
757      return text
758    elif getFilesThread.isAlive() == False: #TODO: send an alert that download is complete
759      text = "Download complete."
760      if self.notificateOnce == True:
761        self.sendMessage('notification:Download complete.#10')
762      self.notificateOnce = False
763      return text
764
765  def getSizeText(self, sizeThread):
766    """return a string describing status of the size counting threads"""
767    tileCount = len(self.currentDownloadList)
768    if tileCount == 0:
769      return ""
770    if sizeThread == None:
771      return ("Total size of tiles is unknown (click to compute).")
772    elif sizeThread.isAlive() == True:
773      totalTileCount = sizeThread.urlCount
774      currentTileCount = sizeThread.processed
775      currentSize = sizeThread.totalSize/(1048576) # = 1024.0*1024.0
776      text = "Checking: %d of %d tiles complete(%1.0f MB)" % (currentTileCount, totalTileCount, currentSize)
777      return text
778    elif sizeThread.isAlive() == False:
779      sizeInMB = sizeThread.totalSize/(1024.0*1024.0)
780      text = "Total size for download: %1.2f MB" % (sizeInMB)
781      return text
782
783  def getFreeSpaceText(self):
784    """return a string describing the space available on the filesystem where the tilefolder is"""
785    path = self.get('tileFolder', None)
786    f = os.statvfs(path)
787    freeSpaceInBytes = (f.f_bsize * f.f_bavail)
788    freeSpaceInMB = freeSpaceInBytes/(1024.0*1024.0)
789    text = "Free space available: %1.1f MB" % freeSpaceInMB
790    return text
791
792
793  def showText(self,cr,text,x,y,widthLimit=None,fontsize=40):
794    if(text):
795      cr.set_font_size(fontsize)
796      stats = cr.text_extents(text)
797      (textwidth, textheight) = stats[2:4]
798
799      if(widthLimit and textwidth > widthLimit):
800        cr.set_font_size(fontsize * widthLimit / textwidth)
801        stats = cr.text_extents(text)
802        (textwidth, textheight) = stats[2:4]
803
804      cr.move_to(x, y+textheight)
805      cr.show_text(text)
806
807  def sendMessage(self,message):
808    m = self.m.get("messages", None)
809    if(m != None):
810      print "mapData: Sending message: " + message
811      m.routeMessage(message)
812    else:
813      print "mapData: No message handler, cant send message."
814
815  def tilesetSvgSnippet(self, f, tileset, colour):
816    for tile in tileset:
817      (x,y) = [int(a) for a in tile.split(",")]
818      f.write("<rect width=\"1\" height=\"1\" x=\"%d\" y=\"%d\" style=\"fill:%s;stroke:#000000;stroke-width:0.05;\" id=\"rect2160\" />\n" % (x,y, colour))
819 
820  def routeSvgSnippet(self, f, route):
821    path = None
822    for pos in route:
823      (lat,lon) = pos
824      (x,y) = latlon2xy(lat, lon, 15)
825      if(path == None):
826        path = "M %f,%f" % (x,y)
827      else:
828        path += " L %f,%f" % (x,y)
829
830    f.write("<path       style=\"fill:none; stroke:white; stroke-width:0.12px;\" d=\"%s\"        id=\"inner\" />\n" % path)
831
832    f.write("<path       style=\"fill:none; stroke:yellow; stroke-width:0.06px;\" d=\"%s\"        id=\"outer\" />\n" % path)
833     
834
835  def tilesetToSvg(self, tilesets, route, filename):
836    f = open(filename, "w")
837    f.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n")
838    f.write("<svg\n   xmlns:svg=\"http://www.w3.org/2000/svg\"\n   xmlns=\"http://www.w3.org/2000/svg\"\n   version=\"1.0\"\n   width=\"1000\"\n   height=\"1000\"   id=\"svg2\">\n")
839
840    print "Creating SVG"
841    f.write("  <g id=\"layer1\">\n")
842    colours = ['red','#FF8000','yellow','green','blue','#808080','black']
843    for tileset in tilesets:
844      colour = colours.pop(0)
845      print " - tileset %s"% colour
846      self.tilesetSvgSnippet(f,tileset, colour)
847    f.write("</g>\n")
848   
849    if(route):
850      f.write("  <g id=\"route\">\n")
851      print " - route"
852      self.routeSvgSnippet(f, route)
853      f.write("</g>\n")
854     
855    f.write("</svg>\n")
856
857  def shutdown(self):
858    if self.sizeThread:
859      try:
860        self.sizeThread.quit=True
861      except:
862        print "error while shutting down size thread"
863
864    if self.getFilesThread:
865      try:
866        self.getFilesThread.quit=True
867      except:
868        print "error while shutting down files thread"
869
870
871if(__name__ == "__main__"):
872  from sample_route import *
873  route = getSampleRoute()
874  a = mapData({}, {})
875
876  if(0): # spirals
877    for d in range(1,10):
878      # Create a spiral of tiles, radius d
879      tileset = a.spiral(100,100,0, d)
880      print "Spiral of width %d = %d locations" %(d,len(tileset))
881
882      # Convert array to dictionary
883      keys = {}
884      count = 0
885      for tile in tileset:
886        (x,y,z) = tile
887        key = "%d,%d" % (x,y)
888        keys[key] = " " * (5-len(str(count))) + str(count)
889        count += 1
890
891      # Print grid of values to a textfile
892      f = open("tiles_%d.txt" % d, "w")
893      for y in range(100 - d - 2, 100 + d + 2):
894        for x in range(100 - d - 2, 100 + d + 2):
895          key = "%d,%d" % (x,y)
896          val = keys.get(key, " ")
897          f.write("%s\t" % val)
898        f.write("\n")
899      f.close()
900         
901  # Load a sample route, and try expanding it
902  if(1):
903    tileset = a.listTiles(route)
904    from time import time
905    print "Route covers %d tiles" % len(tileset)
906
907
908    a.tilesetToSvg([tileset], route, "route.svg")
909       
910    if(1):
911      tilesets = []
912      for i in range(0,14,2):
913        start = time()
914        result = a.expand(tileset,i)
915        dt = time() - start
916        print " expand by %d to get %d tiles (took %1.3f sec)" % (i,len(result), dt)
917        tilesets.insert(0,result)
918
919      a.tilesetToSvg(tilesets, route, "expand.svg" )
Note: See TracBrowser for help on using the repository browser.