Changeset 3f54b2b


Ignore:
Timestamp:
07/23/11 15:08:58 (3 years ago)
Author:
xkolman2 <xkolman2@…>
Branches:
master
Children:
c29aebd
Parents:
5b45cf4
git-author:
xkolman2 <xkolman2@…> (07/23/11 15:08:58)
git-committer:
xkolman2 <xkolman2@…> (07/23/11 15:08:58)
Message:

clean up the tile handling code a bit
store tiles outside of /opt/modrana
use os.path.join for all path joining operations

git-svn-id: https://nlp.fi.muni.cz/svn/gps_navigace/trunk@472 0858a4d0-ffff-46e5-938e-62b5ecb34222

Location:
modules
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • modules/mod_mapData.py

    r81019b0 r3f54b2b  
    2424import time 
    2525import os 
    26 import sys 
    2726import geo 
    2827from threading import Thread 
    2928import threading 
    3029import urllib3 
    31 import random 
    3230#import traceback 
    3331import modrana_utils 
    3432 
    35  
    36  
     33# socket timeout 
    3734import socket 
    3835timeout = 30 # this sets timeout for all sockets 
    3936socket.setdefaulttimeout(timeout) 
    40  
    4137 
    4238 
     
    4945its magic number (it is probably an error response webpage \ 
    5046returned by the server)" 
    51 #         message = "the downloaded tile is not an image as per " 
    52 #         message+= "its magic number (it is probably an error response webpage " 
    53 #         message+= "returned by the server)" 
    5447 
    5548         return message 
    5649          
    57  
    58 #from modules.pyrender.tilenames import xy2latlon 
    59 import sys 
    60 if(__name__ == '__main__'): 
    61   sys.path.append('pyroutelib2') 
    62 else: 
    63   sys.path.append('modules/pyroutelib2') 
    64  
    65 import tiledata 
    66  
    6750def getModule(m,d,i): 
    6851  return(mapData(m,d,i)) 
     
    8770    updated during batch, in seconds""" 
    8871    self.batchInfoUpdateInterval = 0.5 
    89  
     72    self.mapFolderPath = None 
     73 
     74  def firstTime(self): 
     75    # cache the map folder path 
     76    options = self.m.get('options', None) 
     77    if options: 
     78      self.mapFolderPath = options.getMapFolderPath() 
     79 
     80  def _getTileFolderPath(self): 
     81    """return path to the map folder""" 
     82    return self.mapFolderPath 
    9083 
    9184  def listTiles(self, route): 
     
    107100    print "Checking if there are duplicated tiles" 
    108101    start = clock() 
    109 #    self.currentTilesToGet = tilesToDownload # this is for displaying the tiles for debugging reasons 
    110     tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles 
     102    tileFolder = self._getTileFolderPath() # where should we store the downloaded tiles 
     103    if tileFolder == None: 
     104      print "mapData: tile folder path unknown or unusable" 
     105      return [] 
    111106    layer = self.get('layer', None) # TODO: manual layer setting 
    112107    maplayers = self.get('maplayers', None) # a distionary describing supported maplayers 
    113108    extension = maplayers[layer]['type'] # what is the extension for the current layer ? 
    114109    folderPrefix = maplayers[layer]['folderPrefix'] # what is the extension for the current layer ? 
    115 #    alreadyDownloadedTiles = set(os.listdir(tileFolderPath)) # already dowloaded tiles 
    116110 
    117111    mapTiles = self.m.get('mapTiles', None) 
     
    124118      if not os.path.exists(filePath): # we dont have this file 
    125119        neededTiles.append(tile) 
    126 #      if not os.path.exists(filePath): # we dont have this file 
    127 #        url = self.getTileUrl(x, y, z, layer) # generate url 
    128 #        urlsAndFilenames.append((url,filePath)) # store url and path 
    129120 
    130121    print "Downloading %d new tiles." % len(neededTiles) 
     
    135126  def getTileUrlAndPath(self,x,y,z,layer): 
    136127    mapTiles = self.m.get('mapTiles', None) 
    137     tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles 
     128    tileFolder = self._getTileFolderPath() # where should we store the downloaded tiles 
    138129    maplayers = self.get('maplayers', None) # a distionary describing supported maplayers 
    139130    extension = maplayers[layer]['type'] # what is the extension for the current layer ? 
     
    149140    """ 
    150141 
    151     tileFolder = self.get('tileFolder', None) # where should we store the downloaded tiles 
     142    tileFolder = self._getTileFolderPath() # where should we store the downloaded tiles 
    152143    print "tiles for the batch will be downloaded to: %s" % tileFolder 
    153144 
     
    687678        dl = False 
    688679        try: 
    689  
    690  #TESTING ONLY ! - random error generator 
    691 #          r = random.randint(1, 6) 
    692 #          if r == 6: 
    693 #            "BOOM"/2 
    694680 
    695681          dlSize = self.saveTileForURL(item) 
     
    1004990  def getFreeSpaceText(self): 
    1005991    """return a string describing the space available on the filesystem where the tilefolder is""" 
    1006     path = self.get('tileFolder', None) 
     992    path = self._getTileFolderPath() 
    1007993    f = os.statvfs(path) 
    1008994    freeSpaceInBytes = (f.f_bsize * f.f_bavail) 
  • modules/mod_mapTiles.py

    r35f0ed7 r3f54b2b  
    2222from threading import Thread 
    2323import threading 
    24 import Queue 
     24import os 
    2525import traceback 
    2626import cairo 
     
    137137    self.mapViewModule = None 
    138138 
     139    self.mapFolderPath = None 
     140 
    139141  def firstTime(self): 
    140142    self.mapViewModule = self.m.get('mapView', None) 
     
    142144    self.modrana.watch('mapScale', self._updateScalingCB) 
    143145    self.modrana.watch('z', self._updateScalingCB) 
     146 
     147    # cache the map folder path 
     148    options = self.m.get('options', None) 
     149    if options: 
     150      self.mapFolderPath = options.getMapFolderPath() 
    144151 
    145152  def _updateScalingCB(self, key=None, oldValue=None, newValue=None): 
     
    191198                if self.downloadRequestPool: 
    192199                  request = self.downloadRequestPool.pop() # download most recent requests first 
    193                   (name,x,y,z,layer,layerPrefix,layerType, filename, folder, timestamp) = request 
     200                  (name,x,y,z,layer,layerPrefix,layerType, filename, timestamp) = request 
    194201                  # start a new thread 
    195                   self.threads[name] = self.TileDownloader(name,x,y,z,layer,layerPrefix,layerType, filename, folder, self) 
     202                  self.threads[name] = self.TileDownloader(name,x,y,z,layer,layerPrefix,layerType, filename, self) 
    196203                  self.threads[name].daemon = True 
    197204                  self.threads[name].start() 
     
    277284      print("mapTiles: exception in idle loader\n", e) 
    278285      self.idleLoaderActive = False 
     286      traceback.print_exc() 
    279287      return False 
    280288 
     
    703711 
    704712    # Image not found anywhere locally - resort to downloading it 
    705     filename = self.getTileFolderPath() + (self.getImagePath(x,y,z,layerPrefix, layerType)) 
     713    filename = os.path.join(self._getTileFolderPath(), (self.getImagePath(x,y,z,layerPrefix, layerType))) 
    706714    sprint("image not found locally - trying to download") 
    707715 
     
    713721      that here is a new download request""" 
    714722      with self.threadlListCondition: 
    715         folder = self.getTileFolderPath() + self.getImageFolder(x, z, layerPrefix) # target folder 
    716723        timestamp = time.time() 
    717         request = (name,x,y,z,layer,layerPrefix,layerType, filename, folder, timestamp) 
     724        request = (name,x,y,z,layer,layerPrefix,layerType, filename, timestamp) 
    718725 
    719726        with self.imagesLock: # display the "Waiting for download slot..." status tile 
     
    844851    """Get a unique name for a tile image 
    845852    (suitable for use as part of filenames, dictionary keys, etc)""" 
    846     return("%s/%d/%d/%d.%s" % (prefix,z,x,y,extension)) 
     853#    return("%s/%d/%d/%d.%s" % (prefix,z,x,y,extension)) 
     854    return os.path.join(prefix, str(z), str(x), "%d.%s" % (y,extension) ) 
    847855   
    848856  def getImageFolder(self,x,z,prefix): 
     
    851859    return("%s/%d/%d" % (prefix,z,x)) 
    852860 
    853   def getTileFolderPath(self): 
     861  def _getTileFolderPath(self): 
    854862    """helper function that returns path to the tile folder""" 
    855     return self.get('tileFolder', 'cache/images') 
     863    return self.mapFolderPath 
    856864 
    857865  def imageY(self, z,extension): 
     
    881889  class TileDownloader(Thread): 
    882890    """Downloads an image (in a thread)""" 
    883     def __init__(self,name, x,y,z,layer,layerName,layerType,filename, folder, callback): 
     891    def __init__(self,name, x,y,z,layer,layerName,layerType,filename, callback): 
    884892      Thread.__init__(self) 
    885893      self.name = name 
     
    890898      self.layerName = layerName 
    891899      self.layerType = layerType 
    892       self.folder = folder 
    893900      self.finished = 0 
    894901      self.filename = filename 
     
    903910          self.z, 
    904911          self.layer, 
    905           self.filename, 
    906           self.folder) 
     912          self.filename) 
    907913        self.finished = 1 
    908914 
     
    963969        traceback.print_exc() 
    964970 
    965     def downloadTile(self,name,x,y,z,layer,filename, folder): 
     971    def downloadTile(self,name,x,y,z,layer,filename): 
    966972      """Downloads an image""" 
    967973      url = getTileUrl(x,y,z,layer) 
     
    9971003      m = self.callback.m.get('storeTiles', None) 
    9981004      if m: 
    999         m.automaticStoreTile(content, self.layerName, self.z, self.x, self.y, self.layerType, filename, folder, fromThread = True) 
     1005        m.automaticStoreTile(content, self.layerName, self.z, self.x, self.y, self.layerType, filename, fromThread = True) 
    10001006 
    10011007def getTileUrl(x,y,z,layer): #TODO: share this with mapData 
  • modules/mod_storeTiles.py

    r50ed5e8 r3f54b2b  
    297297      mapTiles = self.m.get('mapTiles', None) 
    298298      if mapTiles: 
    299         tileFolderPath = mapTiles.getTileFolderPath() 
     299        tileFolderPath = mapTiles._getTileFolderPath() 
    300300        layerFolderAndTileFilename = mapTiles.getImagePath(x,y,z,folderPrefix, extension) 
    301         tilePath = tileFolderPath + layerFolderAndTileFilename 
     301        tilePath = os.path.join(tileFolderPath, layerFolderAndTileFilename) 
    302302        if os.path.exists(tilePath): 
    303303          # load the file to pixbuf and return it 
     
    411411 
    412412 
    413   def automaticStoreTile(self, tile, folderPrefix, z, x, y, extension, filename, folder, fromThread = False): 
     413  def automaticStoreTile(self, tile, folderPrefix, z, x, y, extension, filename, fromThread = False): 
    414414    """store a tile to a file or db, depending on the current setting""" 
    415415 
     
    419419      self.sqliteTileQueue.put((tile, folderPrefix, z, x, y, extension, filename, folder), block=True, timeout=20) 
    420420    else: # we are storing to the filesystem 
    421       if not os.path.exists(folder): # does it exist ? 
     421      # get the folder path 
     422      (folderPath, tail) = os.path.split(filename) 
     423      if not os.path.exists(folderPath): # does it exist ? 
    422424        try: 
    423           os.makedirs(folder) # create the folder 
     425          os.makedirs(folderPath) # create the folder 
    424426        except: 
    425           print "mapTiles: cant crate folder %s for %s" % (folder,filename) 
    426       f = open(filename, 'w') # write the tile to file 
     427          print "mapTiles: cant create folder %s for %s" % (folderPath,filename) 
     428 
     429      f = open(filename, 'w') # write the tile to a file 
    427430      f.write(tile) 
    428431      f.close() 
Note: See TracChangeset for help on using the changeset viewer.