diff --git a/advancedcaching/abstractmap.py b/advancedcaching/abstractmap.py index 856aa67..823e518 100644 --- a/advancedcaching/abstractmap.py +++ b/advancedcaching/abstractmap.py @@ -20,16 +20,14 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # - -import openstreetmap - import logging -logger = logging.getLogger('abstractmap') -import geo import math +from advancedcaching import geo, openstreetmap +logger = logging.getLogger('abstractmap') + class AbstractMap(object): MAP_FACTOR = 0 diff --git a/advancedcaching/cachedownloader.py b/advancedcaching/cachedownloader.py index 9944473..5aaeb5c 100644 --- a/advancedcaching/cachedownloader.py +++ b/advancedcaching/cachedownloader.py @@ -20,36 +20,54 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -VERSION = 33 +VERSION = 34 VERSION_DATE = '2012-09-11' +import gobject import logging -logger = logging.getLogger('cachedownloader') +import os +import re +import threading + +from lxml.html import fromstring, tostring +from urlparse import urlparse try: import json json.dumps except (ImportError, AttributeError): - import simplejson as json -from geocaching import GeocacheCoordinate -import geo -import os -import threading -import re -import gobject -from utils import HTMLManipulations -from lxml.html import fromstring, tostring, submit_form + import simplejson as json + +from advancedcaching import geo +from advancedcaching.constants import TYPE_UNKNOWN, TYPE_REGULAR, GC_TYPE_MAP +from advancedcaching.geocaching import GeocacheCoordinate +from advancedcaching.utils import HTMLManipulations + + +logger = logging.getLogger('cachedownloader') #ugly workaround... user_token = [None] +MESSAGE_DISABLED = 'This cache is temporarily unavailable. Read the logs below to read the status for this cache.' +MESSAGE_ARCHIVED = 'This cache has been archived, but is available for viewing for archival purposes.' + + +def url_basename(url): + """ + Returns last part of URL path without filename extension. + """ + chunks = urlparse(url) + filename = chunks[2].rsplit('/', 1)[1] + return os.path.splitext(filename)[0] + class CacheDownloader(gobject.GObject): __gsignals__ = { 'progress' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (str, float, float, )), 'download-error' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)), - 'already-downloading-error' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)) - 'need-auth-data' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (str)), + 'already-downloading-error' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)), + 'need-auth-data' : (gobject.SIGNAL_RUN_FIRST, gobject.TYPE_NONE, (str, )), } lock = threading.Lock() @@ -65,15 +83,14 @@ def __init__(self, downloader, path = None, download_images = True): os.mkdir(path) except: raise Exception("Path does not exist: %s" % path) - + def update_userdata(self, username = None, password = None): ''' Change the settings for the user data and reset the cookies afterwards so that a new login will be performed. - + #todo: check if username can actually be change by calling this method. - + ''' - if username != None: self.username = username if password != None: @@ -92,7 +109,7 @@ def update_coordinates(self, coordinates, num_logs = 20): c.append(u) i += 1 return c - + # Update a single coordinate def update_coordinate(self, coordinate, num_logs = 20, progress_min = 0.0, progress_max = 1.0, progress_all = 1.0): if not CacheDownloader.lock.acquire(False): @@ -109,13 +126,13 @@ def update_coordinate(self, coordinate, num_logs = 20, progress_min = 0.0, progr CacheDownloader.lock.release() return u - # Retrieve geocaches in the bounding box defined by location + # Retrieve geocaches in the bounding box defined by location def get_overview(self, location, get_geocache_callback, skip_callback = None): if not CacheDownloader.lock.acquire(False): self.emit('already-downloading-error', Exception("There's a download in progress. Please wait.")) logger.warning("Download in progress") return - + try: points = self._get_overview(location, get_geocache_callback, skip_callback = skip_callback) except Exception, e: @@ -126,51 +143,49 @@ def get_overview(self, location, get_geocache_callback, skip_callback = None): CacheDownloader.lock.release() return points - + # Upload one or more fieldnotes - def upload_fieldnotes(self, geocaches, upload_as_logs = False): - + def upload_fieldnotes_and_logs(self, geocaches): try: - if not upload_as_logs: - self._upload_fieldnotes(geocaches) - else: - self._upload_logs(geocaches) + success = self._upload_fieldnotes_and_logs(geocaches) except Exception, e: + logger.exception(e) self.emit('download-error', e) return [] - return geocaches + + return success """ class OpenCachingComCacheDownloader(CacheDownloader): ''' New Backend for Opencaching.com - + ''' - + def _get_overview(self, location, get_geocache_callback, skip_callback = None): def _update_coordinate(self, coordinate, num_logs = 20, progress_min = 0.0, progress_max = 1.0, progress_all = 1.0): - - -""" - - + + +""" + + class GeocachingComCacheDownloader(CacheDownloader): - MAX_REC_DEPTH = 3 MAX_DOWNLOAD_NUM = 800 + TRANS_FIELDNOTE_TYPE = { + GeocacheCoordinate.LOG_AS_FOUND: "Found it", + GeocacheCoordinate.LOG_AS_NOTFOUND: "Didn't find it", + GeocacheCoordinate.LOG_AS_NOTE: "Write note" + } - CTIDS = { - 2:GeocacheCoordinate.TYPE_REGULAR, - 3:GeocacheCoordinate.TYPE_MULTI, - 4:GeocacheCoordinate.TYPE_VIRTUAL, - 6:GeocacheCoordinate.TYPE_EVENT, - 8:GeocacheCoordinate.TYPE_MYSTERY, - 11:GeocacheCoordinate.TYPE_WEBCAM, - 137:GeocacheCoordinate.TYPE_EARTH + TRANS_LOG_TYPE = { + GeocacheCoordinate.LOG_AS_FOUND: 2, + GeocacheCoordinate.LOG_AS_NOTFOUND: 3, + GeocacheCoordinate.LOG_AS_NOTE: 4 } - + # URL for log pages; fetches 10 logs by default LOGBOOK_URL = 'http://www.geocaching.com/seek/geocache.logbook?tkn=%s&idx=%d&num=10&decrypt=true' OVERVIEW_URL = 'http://www.geocaching.com/seek/nearest.aspx?lat=%f&lng=%f&dist=%f' @@ -181,19 +196,29 @@ class GeocachingComCacheDownloader(CacheDownloader): NEAREST_URL = 'http://www.geocaching.com/seek/nearest.aspx' USER_TOKEN_URL = 'http://www.geocaching.com/map/default.aspx?lat=6&lng=9' UPLOAD_FIELDNOTES_URL = 'http://www.geocaching.com/my/uploadfieldnotes.aspx' - + UPLOAD_LOG_URL = 'http://www.geocaching.com/seek/log.aspx?wp=%s' + def __init__(self, downloader, path = None, download_images = True): CacheDownloader.__init__(self, downloader, path, download_images) self.downloader.allow_minified_answers = True - - def __download(self, url, values = None, data = None, raw = False): - sucess = False - while not success: + + def __download(self, url, values = None, data = None, raw = False, skip_login = False): + ''' + Download a resource. If raw is True, return a file like object. If raw is False, check whether the user is logged in and if not so, perform a log in. Then, return an lxml document. + + ''' + response = self.downloader.get_reader(url, data, values) + if raw: + return response.read() + doc = self.__read_document(response) + if skip_login or self.__check_login(doc): + return doc + else: + self.__perform_login() response = self.downloader.get_reader(url, data, values) - if raw: - return response.read() doc = self.__read_document(response) - success = self.__check_and_perform_login(doc) + if not self.__check_login(doc): + raise Exception("Cannot login, for whatever reason.") return doc def __read_document(self, page): @@ -216,11 +241,11 @@ def _get_overview(self, location, get_geocache_callback, skip_callback = None): if dist > 100: raise Exception("Please select a smaller part of the map!") url = self.OVERVIEW_URL % (center.lat, center.lon, dist) - + self.emit("progress", "Fetching list", 0, 1) - + doc = self.__download(url) - + cont = True wpts = [] page_last = 0 # Stores the "old" value of the page counter; If it doesn't increment, abort! @@ -239,20 +264,18 @@ def _get_overview(self, location, get_geocache_callback, skip_callback = None): logger.info("We are at page %d of %d, total %d geocaches" % (page_current, page_max, count)) if count > self.MAX_DOWNLOAD_NUM: raise Exception("%d geocaches found, please select a smaller part of the map!" % count) - - + + # Extract waypoint information from the page - w = [( - # Get the GUID from the link - x.getparent().getchildren()[0].get('href').split('guid=')[1], - # See whether this cache was found or not - 'TertiaryRow' in x.getparent().getparent().get('class'), - # Get the GCID from the text - x.text_content().split('|')[1].strip() - ) for x in doc.cssselect(".SearchResultsTable .Merge .small")] + # (GUID, found, disabled, GCID) + w = [(x.getparent().getchildren()[0].get('href').split('guid=')[1], + 'TertiaryRow' in x.getparent().getparent().get('class'), + 'Strike' in x.getparent().getchildren()[0].get('class'), + x.text_content().split('|')[1].strip()) + for x in doc.cssselect(".SearchResultsTable .Merge .small")] wpts += w - - cont = False + + cont = False # There are more pages... if page_current < page_max: from urllib import urlencode @@ -263,18 +286,18 @@ def _get_overview(self, location, get_geocache_callback, skip_callback = None): action = self.SEEK_URL % doc.forms[0].action logger.info("Retrieving next page!") self.emit("progress", "Fetching list (%d of %d)" % (page_current + 1, page_max), page_current, page_max) - doc = self.__download(action, data=('application/x-www-form-urlencoded', values)) - + doc = self.__download(action, data=values) + cont = True - + # Now, split wpts into three groups: points_that_need_downloading = [] # Geocaches that need to be downloaded points_finished = [] # Geocaches that only need to be updated in the database # and Geocaches which don't need any update (they will be removed) - for guid, found, id in wpts: + for guid, found, disabled, id in wpts: # Check if geocache exists in DB coordinate = get_geocache_callback(id) - + if coordinate == None: # If the coordinate was not in the DB if skip_callback != None and skip_callback(None, found): @@ -285,28 +308,35 @@ def _get_overview(self, location, get_geocache_callback, skip_callback = None): points_that_need_downloading.append((guid, found, id, GeocacheCoordinate(-1, -1, id))) logger.info("Downloading %s. It was not in the DB." % id) continue - + + cache_changed = False + needs_update = False + # Only active and disabled caches are listed. + if disabled: + new_status = GeocacheCoordinate.STATUS_DISABLED + else: + new_status = GeocacheCoordinate.STATUS_NORMAL + if coordinate.status != new_status: + coordinate.status = new_status + cache_changed = True + if coordinate.found != found: coordinate.found = found + cache_changed = True needs_update = True - else: - needs_update = False - + if skip_callback != None and skip_callback(coordinate, found): if not needs_update: - logger.info("Skipping %s. It was in the DB, but its found status was correct." % id) - continue - # If the coordinate is to be skipped, put it into points_finished anyway - # because the found status was updated - points_finished.append(coordinate) - logger.info("Updating %s. It was in the DB, but its found status was not correct." % id) + logger.info("Skipping %s. It was in the DB." % id) + if cache_changed: + points_finished.append(coordinate) + logger.info("Updating %s. Its found status was not correct." % id) continue logger.info("Downloading %s. It was in the DB, but it was not to be skipped." % id) points_that_need_downloading.append((guid, found, id, coordinate)) - - - # Download the geocaches using the print preview + + # Download the geocaches using the print preview i = 0 for guid, found, id, coordinate in points_that_need_downloading: i += 1 @@ -315,40 +345,50 @@ def _get_overview(self, location, get_geocache_callback, skip_callback = None): self.emit("progress", "Geocache %d of %d" % (i, len(points_that_need_downloading)), i, len(points_that_need_downloading)) logger.info("Downloading %s..." % id) url = self.PRINT_PREVIEW_URL % guid - - doc = self.__download(url) - result = self.__parse_cache_page_print(doc, coordinate, num_logs = 20) + + doc = self.__download(url, skip_login = True) # login check doesn't work with print preview, therefore skipping it. + try: + result = self.__parse_cache_page_print(doc, coordinate, num_logs = 20) + except (ValueError, TypeError, LookupError): + # Ignore parsing errors, but continue with other caches. + logging.warning("Skipping cache %s: error in parsing details.", id) + continue if result != None and result.lat != -1: points_finished.append(result) - + return points_finished - + def _update_coordinate(self, coordinate, num_logs = 20, progress_min = 0.0, progress_max = 1.0, progress_all = 1.0): coordinate = coordinate.clone() - + logger.debug("_update_coordinate, pmin = %f, pmax = %f." % (progress_min, progress_max)) - - # Progress should be displayed in the range between progress_min and progress_max. + + # Progress should be displayed in the range between progress_min and progress_max. self.emit('progress', "Downloading %s" % coordinate.name, progress_min, progress_all) - + url = self.DETAILS_URL % coordinate.name - - doc = self.__download(url) - - return self.__parse_cache_page(response, coordinate, num_logs, progress_min = progress_min, progress_max = progress_max, progress_all = progress_all) - - def __check_and_perform_login(self, doc): + doc = self.__download(url) + + return self.__parse_cache_page(doc, coordinate, num_logs, progress_min = progress_min, progress_max = progress_max, progress_all = progress_all) + + def __check_login(self, doc): ''' - Checks the document doc to see whether we are logged in or not. If not, performs the login. - - Returns whether doc was logged in. - + Checks the document doc to see whether we are logged in or not. + + Returns whether doc was logged in, i.e., whether doc was a resource which was retrieved with valid credentials or not. + ''' if len(doc.cssselect('.SignedInText')) > 0: - logger.info("Probably still signed in.") + logger.debug("User is signed in.") return True - + return False + + def __perform_login(self): + ''' + Perform a login. + + ''' values = {'ctl00$ContentBody$tbUsername': self.username, 'ctl00$ContentBody$tbPassword': self.password, 'ctl00$ContentBody$cbRememberMe': 'on', @@ -356,49 +396,42 @@ def __check_and_perform_login(self, doc): '__EVENTTARGET': '', '__EVENTARGUMENT': '' } - + # Perform the login + logger.debug("Performing login.") request = self.downloader.get_reader(GeocachingComCacheDownloader.LOGIN_URL, values) doc = self.__read_document(request) - + + if doc.get_element_by_id('ctl00_liNavJoin', None) == None: + logger.debug("Sign in succeded.") + return + if doc.get_element_by_id('ctl00_liNavProfile', None) == None: + logger.debug("Sign in falied.") self.emit('need-auth-data', self.__class__.__name__) raise Exception("Wrong username or password!") - elif doc.get_element_by_id('ctl00_liNavJoin', None) == None: - logger.info("Great success.") - return False + raise Exception("Name/Password MAY be correct, but I encountered unexpected data while logging in.") - - def __parse_cache_page(self, cache_page, coordinate, num_logs, download_images = True, progress_min = 0.0, progress_max = 1.0, progress_all = 1.0): + + def __parse_cache_page(self, doc, coordinate, num_logs, download_images = True, progress_min = 0.0, progress_max = 1.0, progress_all = 1.0): logger.debug("Start parsing, pmin = %f, pmax = %f." % (progress_min, progress_max)) - doc = self.__read_document(cache_page) - - # Basename - Image name without path and extension - def basename(url): - return url.split('/')[-1].split('.')[0] - # Title try: coordinate.title = doc.cssselect('meta[name="og:title"]')[0].get('content') except Exception, e: logger.error("Could not find title - cache is probably unpublished!") raise Exception("Geocache not found.") - - # Type - try: - t = int(basename(doc.cssselect('.cacheImage img')[0].get('src')).split('.')[0]) - coordinate.type = self.CTIDS[t] if t in self.CTIDS else GeocacheCoordinate.TYPE_UNKNOWN - except Exception, e: - logger.error("Could not find type!") - raise e - + + # Type + coordinate.type = self._parse_type(doc, '.cacheImage img') + # Website try: coordinate.websitelink = doc.get_element_by_id('ctl00_ContentBody_uxCacheUrl').get('href') except KeyError, e: logger.info("No website link found, skipping.") coordinate.websitelink = '' - + # Short Description - Long Desc. is added after the image handling (see below) try: coordinate.shortdesc = doc.get_element_by_id('ctl00_ContentBody_ShortDescription').text_content() @@ -417,22 +450,25 @@ def basename(url): except Exception, e: logger.error("Could not parse this coordinate: %r" % text) raise e - - # Size + + # Size try: # src is URL to image of the cache size # src.split('/')[-1].split('.')[0] is the basename minus extension - coordinate.size = self._handle_size(basename(doc.cssselect('.CacheSize p span img')[0].get('src'))) + coordinate.size = self._handle_size(url_basename(doc.cssselect('.CacheSize p span img')[0].get('src'))) except Exception, e: logger.error("Could not find/parse size string") raise e - + # Terrain/Difficulty try: - coordinate.difficulty, coordinate.terrain = [self._handle_stars(basename(x.get('src'))) for x in doc.cssselect('.CacheStarImgs span img')] + coordinate.difficulty, coordinate.terrain = [self._handle_stars(url_basename(x.get('src'))) for x in doc.cssselect('.CacheStarImgs span img')] except Exception, e: logger.error("Could not find/parse star ratings") - + + # Status + coordinate.status = self._parse_page_status(doc) + # Hint(s) try: hint = doc.get_element_by_id('div_hint') @@ -440,14 +476,14 @@ def basename(url): except KeyError, e: logger.info("Hint not found!") coordinate.hints = '' - + # Owner try: coordinate.owner = doc.cssselect('#cacheDetails .minorCacheDetails a')[0].text_content() except Exception, e: logger.error("Owner not found!") raise e - + # Waypoints waypoints = [] w = {} @@ -457,7 +493,7 @@ def basename(url): w['name'] = x[5].text_content().strip() try: coord = geo.try_parse_coordinate(x[6].text_content().strip()) - w['lat'], w['lon'] = coord.lat, coord.lon + w['lat'], w['lon'] = coord.lat, coord.lon except Exception, e: w['lat'], w['lon'] = -1, -1 else: @@ -465,7 +501,7 @@ def basename(url): waypoints += [w] w = {} coordinate.set_waypoints(waypoints) - + # User token and Logs userToken = '' for x in doc.cssselect('script'): @@ -476,13 +512,13 @@ def basename(url): userToken = re.sub("(?s).*userToken = '", '', s) userToken = re.sub("(?s)'.*", '', userToken) logger.debug("userToken: %s" % userToken) - + self.emit('progress', 'Fetching logs', progress_min + 0.2 * (progress_max - progress_min), progress_all) - + #Ask first page of logs. And same time number of pages - doc = self.__download(self.downloader.get_reader(self.LOGBOOK_URL % (userToken, 1), raw = True) - new_set_of_logs, total_page = self._parse_logs_json(doc) #True=we want also number of page - logs.close() + doc_logs = self.__download(self.LOGBOOK_URL % (userToken, 1), raw = True) + new_set_of_logs, total_page = self._parse_logs_json(doc_logs) #True=we want also number of page + page_of_logs=num_logs/10 #num_logs from parameter (which comes from settings 'download_num_logs') #First page is already handled, so counter starts from 2 @@ -496,9 +532,8 @@ def basename(url): logger.debug("- Progress internal is %f" % progress_internal) logger.debug("- Progress is %f of %f" % (progress_min + progress_internal * (progress_max - progress_min), progress_all)) self.emit('progress', "Logs (%d/%d)" % (counter, upper_limit), progress_min + progress_internal * (progress_max - progress_min), progress_all) - doc = self.__download(self.LOGBOOK_URL % (userToken, counter), raw = True) - new_set_of_logs.extend(self._parse_logs_json(doc)[0]) - logs.close() + doc_logs = self.__download(self.LOGBOOK_URL % (userToken, counter), raw = True) + new_set_of_logs.extend(self._parse_logs_json(doc_logs)[0]) counter += 1 @@ -511,7 +546,7 @@ def basename(url): attr_xml = doc.cssselect('.CacheDetailNavigationWidget.BottomSpacing .WidgetBody img') attributes = self._parse_attributes_from_doc(attr_xml) coordinate.clear_attributes() - for x in attributes: + for x in attributes: coordinate.add_attribute(x) except IndexError: # There are no attributes @@ -519,8 +554,7 @@ def basename(url): except Exception, e: logger.error("Could not find/parse attributes") raise e - ''' - + ''' # Image Handling images = {} @@ -530,8 +564,8 @@ def found_image(url, title): # First, only use the large geocaching.com images if url.startswith('http://img.geocaching.com/cache/') and not url.startswith('http://img.geocaching.com/cache/large/'): url = url.replace('http://img.geocaching.com/cache/', 'http://img.geocaching.com/cache/large/') - - # Then, check if this URL is known + + # Then, check if this URL is known if url in images: # If it is, take the longest available title if len(images[url]['title']) < title: @@ -539,7 +573,7 @@ def found_image(url, title): # Return the previously calculated filename # Obviously, it points to the image from the same URL return images[url]['filename'] - + # If this URL is encountered for the first time, calculate the filename ext = url.rsplit('.', 1)[1] if not re.match('^[a-zA-Z0-9]+$', ext): @@ -551,7 +585,7 @@ def found_image(url, title): # Find Images in the additional image section for x in doc.cssselect('a[rel=lightbox]'): found_image(x.get('href'), x.text_content().strip()) - + # Search images in Description and replace them by a special placeholder # First, extract description... try: @@ -559,7 +593,7 @@ def found_image(url, title): except Exception, e: logger.error("Description could not be found!") raise e - + # Next, search image elements and replace them for element, attribute, link, pos in desc.iterlinks(): if element.tag == 'img': @@ -573,9 +607,9 @@ def found_image(url, title): else: parent[index-1].tail += replacement del parent[index] - + counter = 0 - if download_images: + if download_images: # Download images num = len(images) for url, data in images.items(): @@ -587,7 +621,7 @@ def found_image(url, title): # Prepend local path to filename filename = os.path.join(self.path, data['filename']) logger.info("Downloading %s to %s" % (url, filename)) - + # Download file try: f = open(filename, 'wb') @@ -597,43 +631,53 @@ def found_image(url, title): logger.exception(e) logger.error("Failed to download image from URL %s" % url) counter += 1 - + # And save Images to coordinate images_save = dict([x['filename'], x['title']] for x in images.values()) coordinate.set_images(images_save) - + # Long description coordinate.desc = self._extract_node_contents(desc) - - # Archived status - for log in coordinate.get_logs(): - if log['type'] == GeocacheCoordinate.LOG_TYPE_ENABLED: - break - elif log['type'] == GeocacheCoordinate.LOG_TYPE_DISABLED: - coordinate.status = GeocacheCoordinate.STATUS_DISABLED - break - elif log['type'] == GeocacheCoordinate.LOG_TYPE_ARCHIVED: - coordinate.status = GeocacheCoordinate.STATUS_ARCHIVED - break - else: - coordinate.stats = GeocacheCoordinate.STATUS_NORMAL - + # And finally, set last updated time coordinate.touch_updated() - + logger.debug("End parsing.") return coordinate - + + def _parse_page_status(self, dom): + """ + Parses geocache status from DOM. + """ + element = dom.cssselect('#Content ul.OldWarning li') + if not element: + return GeocacheCoordinate.STATUS_NORMAL + + message = element[0].text_content().strip() + if message == MESSAGE_DISABLED: + return GeocacheCoordinate.STATUS_DISABLED + elif message == MESSAGE_ARCHIVED: + return GeocacheCoordinate.STATUS_ARCHIVED + else: + logger.error("Unknown cache status.") + return GeocacheCoordinate.STATUS_NORMAL + + def _parse_type(self, doc, selector): + img_elements = doc.cssselect(selector) + if len(img_elements) != 1: + raise ValueError('Cache type element not found in document.') + + try: + t_id = int(url_basename(img_elements[0].get('src'))) + except (AttributeError, ValueError, TypeError): + logger.error("Could not find type!") + raise + return GC_TYPE_MAP.get(t_id, TYPE_UNKNOWN) + # This parses the print preview of a geocache # It currently omits images, waypoints and logs. - def __parse_cache_page_print(self, cache_page, coordinate, num_logs): + def __parse_cache_page_print(self, doc, coordinate, num_logs): logger.debug("Start parsing.") - doc = self.__read_document(cache_page) - - # Basename - Image name without path and extension - def basename(url): - return url.split('/')[-1].split('.')[0] - # Title, ID and Owner try: text = doc.cssselect('title')[0].text_content() @@ -643,16 +687,11 @@ def basename(url): except Exception, e: logger.error("Could not find title, id or owner!") logger.exception(e) - raise e - - # Type - try: - t = int(basename(doc.cssselect('#Content h2 img')[0].get('src')).split('.')[0]) - coordinate.type = self.CTIDS[t] if t in self.CTIDS else GeocacheCoordinate.TYPE_UNKNOWN - except Exception, e: - logger.error("Could not find type - probably premium cache!") - return None - + raise e + + # Type + coordinate.type = self._parse_type(doc, '#Content h2 img') + # Short Description - Long Desc. is added after the image handling (see below) try: coordinate.shortdesc = doc.cssselect('#Content .sortables .item-content')[1].text_content().strip() @@ -674,24 +713,24 @@ def basename(url): logger.error("Could not parse this coordinate: %r" % text) logger.exception(e) raise e - - # Size + + # Size try: # src is URL to image of the cache size # src.split('/')[-1].split('.')[0] is the basename minus extension - coordinate.size = self._handle_size(basename(doc.cssselect('.Third .Meta img')[0].get('src'))) + coordinate.size = self._handle_size(url_basename(doc.cssselect('.Third .Meta img')[0].get('src'))) except Exception, e: logger.error("Could not find/parse size string") logger.exception(e) raise e - + # Terrain/Difficulty try: - coordinate.difficulty, coordinate.terrain = [self._handle_stars(basename(x.get('src'))) for x in doc.cssselect('.Third .Meta img')[1:]] + coordinate.difficulty, coordinate.terrain = [self._handle_stars(url_basename(x.get('src'))) for x in doc.cssselect('.Third .Meta img')[1:]] except Exception, e: logger.error("Could not find/parse star ratings") logger.exception(e) - + # Hint(s) try: hint = self._extract_node_contents(doc.cssselect('#uxEncryptedHint')[0]) @@ -699,14 +738,14 @@ def basename(url): except IndexError, e: logger.info("Hint not found!") coordinate.hints = '' - + # Attributes try: attr_xml = doc.cssselect('#Content .sortables .item-content')[5].cssselect('img') attributes = self._parse_attributes_from_doc(attr_xml) logger.debug("Found the following attributes: %r" % attributes) coordinate.clear_attributes() - for x in attributes: + for x in attributes: coordinate.add_attribute(x) except IndexError: # There are no attributes @@ -715,7 +754,7 @@ def basename(url): logger.exception(e) logger.error("Could not find/parse attributes") raise e - + # Extract description... try: @@ -735,7 +774,7 @@ def basename(url): w['name'] = x[5].text_content().strip() try: coord = geo.try_parse_coordinate(x[6].text_content().strip()) - w['lat'], w['lon'] = coord.lat, coord.lon + w['lat'], w['lon'] = coord.lat, coord.lon except Exception, e: w['lat'], w['lon'] = -1, -1 else: @@ -749,13 +788,11 @@ def basename(url): logger.debug("End parsing.") return coordinate - - def _parse_attributes_from_doc(self, imgs): """ Takes a document tree representing the attribute images as input and parses the attributes. Downloads any attribute images which are not in the file system yet. - + """ attributes = [] for x in imgs: @@ -784,92 +821,137 @@ def _parse_attributes_from_doc(self, imgs): # store filename without path to the comma separated string attributes.append(attrib) return attributes - + + def _upload_fieldnotes_and_logs(self, geocaches): + ''' + Upload all fieldnotes and logs of the geocaches given in the first argument and return a list of geocaches, for which uploading worked. + + ''' + upload_fieldnotes = [] + upload_logs = [] + for geocache in geocaches: + if geocache.upload_as == GeocacheCoordinate.UPLOAD_AS_LOG: + upload_logs.append(geocache) + else: + upload_fieldnotes.append(geocache) + + + logger.info("Will try to upload %d fieldnotes and %d logs." % (len(upload_fieldnotes), len(upload_logs))) + + success_fieldnotes = self.__upload_fieldnotes(upload_fieldnotes) + success_logs = self.__upload_logs(upload_logs) + + before = len(geocaches) + after = len(success_fieldnotes) + len(success_logs) + + if after == 0 and before != 0: + self.emit("download-error", Exception("Uploading of fieldnotes and logs failed.")) + elif after < before: + self.emit("download-error", Exception("Uploading failed for some fieldnotes or logs.")) + + return success_fieldnotes + success_logs + # Upload one or more fieldnotes - def _upload_fieldnotes(self, geocaches): + def __upload_fieldnotes(self, geocaches): notes = [] + if len(geocaches) == 0: + return [] logger.info("Preparing fieldnotes (new downloader)...") - for geocache in geocaches: - if geocache.logdate == '': - raise Exception("Illegal Date.") - - if geocache.logas == GeocacheCoordinate.LOG_AS_FOUND: - log = "Found it" - elif geocache.logas == GeocacheCoordinate.LOG_AS_NOTFOUND: - log = "Didn't find it" - elif geocache.logas == GeocacheCoordinate.LOG_AS_NOTE: - log = "Write note" - else: - raise Exception("Illegal status: %s" % geocache.logas) - - text = geocache.fieldnotes.replace('"', "'") - - notes.append('%s,%sT10:00Z,%s,"%s"' % (geocache.name, geocache.logdate, log, text)) - - logger.info("Uploading fieldnotes...") - - self.emit('progress', "Uploading Fieldnotes (Step 1 of 2)", 0, 2) - - # First, download webpage to get the correct viewstate value - doc = self.__download(self.UPLOAD_FIELDNOTES_URL) - # Sometimes this field is not available - if 'ctl00$ContentBody$chkSuppressDate' in doc.forms[0].fields: - doc.forms[0].fields['ctl00$ContentBody$chkSuppressDate'] = '' - values = doc.forms[0].form_values() - values += [('ctl00$ContentBody$btnUpload', 'Upload Field Note')] - content = '\r\n'.join(notes).encode("UTF-16") - - data = self.downloader.encode_multipart_formdata(values, [('ctl00$ContentBody$FieldNoteLoader', 'geocache_visits.txt', content)]) - self.emit('progress', "Uploading Fieldnotes (Step 2 of 2)", 1, 2) - doc = self.__download(self.UPLOAD_FIELDNOTES_URL, data=data) - - # There's no real success/no success message on the website. - # We therefore assume success, if this element is in the response - if doc.get_element_by_id('ctl00_ContentBody_lnkFieldNotes', None) == None: - raise Exception("Something went wrong while uploading the field notes.") - else: - logger.info("Finished upload!") - - # Upload one or more logs - def _upload_logs(self, geocaches): - logger.info("Preparing logs...") - for geocache in geocaches: - if geocache.logdate == '': - raise Exception("Illegal Date.") - - if geocache.logas == GeocacheCoordinate.LOG_AS_FOUND: - log = 2 - elif geocache.logas == GeocacheCoordinate.LOG_AS_NOTFOUND: - log = 3 - elif geocache.logas == GeocacheCoordinate.LOG_AS_NOTE: - log = 4 - else: - raise Exception("Illegal status: %s" % geocache.logas) + try: + for geocache in geocaches: + name, logdate, logtype, fieldnotes = geocache.name, geocache.logdate, geocache.logas, geocache.fieldnotes + if logdate == '': + raise Exception("Illegal Date.") + + try: + logtype_trans = self.TRANS_FIELDNOTE_TYPE[logtype] + except KeyError, e: + raise Exception("Illegal status: %s" % logtype) + + text = fieldnotes.replace('"', "'") + + notes.append('%s,%sT10:00Z,%s,"%s"' % (name, logdate, logtype_trans, text)) + + logger.info("Uploading fieldnotes...") + + self.emit('progress', "Uploading Fieldnotes (Step 1 of 2)", 0, 2) - text = geocache.fieldnotes - year, month, day = geocache.logdate.split('-') - - url = 'http://www.geocaching.com/seek/log.aspx?wp=%s' % geocache.name - # First, download webpage to get the correct viewstate value - doc = self.__download(url) - doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$ddLogType'] = str(log) - doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Day'] = str(int(day)) - doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Month'] = str(int(month)) - doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Year'] = str(int(year)) - doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$uxLogInfo'] = text - values = dict(doc.forms[0].form_values()) - values['ctl00$ContentBody$LogBookPanel1$LogButton'] = doc.get_element_by_id('ctl00_ContentBody_LogBookPanel1_LogButton').get('value') - logger.debug("Field values are %r" % values) - doc = self.__download(url, values=values) - - # There's no real success/no success message on the website. + + doc = self.__download(self.UPLOAD_FIELDNOTES_URL) + # Sometimes this field is not available + if 'ctl00$ContentBody$chkSuppressDate' in doc.forms[0].fields: + doc.forms[0].fields['ctl00$ContentBody$chkSuppressDate'] = '' + values = doc.forms[0].form_values() + values += [('ctl00$ContentBody$btnUpload', 'Upload Field Note')] + content = '\r\n'.join(notes).encode("UTF-16") + + data = self.downloader.encode_multipart_formdata(values, [('ctl00$ContentBody$FieldNoteLoader', 'geocache_visits.txt', content)]) + self.emit('progress', "Uploading Fieldnotes (Step 2 of 2)", 1, 2) + + doc = self.__download(self.UPLOAD_FIELDNOTES_URL, + data = data) + + # There's no real success/no success message on the website. # We therefore assume success, if this element is in the response - if doc.get_element_by_id('ctl00_ContentBody_LogBookPanel1_ViewLogPanel', None) == None: - raise Exception("Something went wrong while uploading the log.") + if doc.get_element_by_id('ctl00_ContentBody_lnkFieldNotes', None) == None: + raise Exception("Something went wrong while uploading the field notes.") else: logger.info("Finished upload!") - + return geocaches + except Exception, e: + logger.exception(e) + + # Upload one or more logs + def __upload_logs(self, geocaches): + if len(geocaches) == 0: + return [] + success = [] + logger.info("Preparing logs...") + i = 0 + for geocache in geocaches: + try: + name, logdate, logtype, text = geocache.name, geocache.logdate, geocache.logas, geocache.fieldnotes + + if logdate == '': + raise Exception("Illegal Date.") + + try: + logtype_trans = self.TRANS_LOG_TYPE[logtype] + except KeyError, e: + raise Exception("Illegal status: %s" % logtype) + + year, month, day = logdate.split('-') + + url = self.UPLOAD_LOG_URL % name + + # First, download webpage to get the correct viewstate value + + self.emit('progress', "Uploading Logs (%d of %d)..." % (i, len(geocaches)), i + 1, len(geocaches)) + + doc = self.__download(url) + doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$ddLogType'] = str(logtype_trans) + doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Day'] = str(int(day)) + doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Month'] = str(int(month)) + doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$DateTimeLogged$Year'] = str(int(year)) + doc.forms[0].fields['ctl00$ContentBody$LogBookPanel1$uxLogInfo'] = text + values = dict(doc.forms[0].form_values()) + values['ctl00$ContentBody$LogBookPanel1$LogButton'] = doc.get_element_by_id('ctl00_ContentBody_LogBookPanel1_LogButton').get('value') + logger.debug("Field values are %r" % values) + doc = self.__download(url, values=values) + + # There's no real success/no success message on the website. + # We therefore assume success, if this element is in the response + if doc.get_element_by_id('ctl00_ContentBody_LogBookPanel1_ViewLogPanel', None) == None: + raise Exception("Something went wrong while uploading the log.") + else: + logger.info("Finished upload!") + i += 1 + success.append(geocache) + except Exception, e: + logger.exception(e) + return success + # Only return the contents of a node, not the node tag itself, as text def _extract_node_contents(self, el): # Alternative solution for this would be: @@ -878,7 +960,7 @@ def _extract_node_contents(self, el): #out = re.sub('<[^>]+>$', '', out) #return out return (el.text if el.text != None else '') + ''.join(unicode(tostring(x, encoding='utf-8', method='html'), 'utf-8') for x in el) - + # Handle size string from basename of the according image def _handle_size(self, sizestring): if sizestring == 'micro': @@ -889,21 +971,21 @@ def _handle_size(self, sizestring): size = 3 elif sizestring == 'large' or sizestring == 'big': size = 4 - else: + else: size = 5 return size - + # Convert stars3_5 to 35, stars4 to 4 (and so on, basename of image of star rating) def _handle_stars(self, stars): return int(stars[5])*10 + (int(stars[7]) if len(stars) > 6 else 0) - + def _handle_hints(self, hints, encrypted = True): hints = HTMLManipulations._strip_html(HTMLManipulations._replace_br(hints)).strip() if encrypted: hints = HTMLManipulations._rot13(hints) hints = re.sub(r'\[([^\]]+)\]', lambda match: HTMLManipulations._rot13(match.group(0)), hints) return hints - + def _parse_logs_json(self, logs): logger.debug("Start json logs parsing") try: @@ -923,13 +1005,13 @@ def _parse_logs_json(self, logs): text = HTMLManipulations._decode_htmlentities(HTMLManipulations._strip_html(HTMLManipulations._replace_br(l['LogText']))) output.append(dict(type=tpe, date=date, finder=finder, text=text)) logger.debug("Read %d log entries" % len(output)) - + total_page = r['pageInfo']['totalPages'] return output,total_page BACKENDS = { 'geocaching-com-new': {'class': GeocachingComCacheDownloader, 'name': 'geocaching.com', 'description': 'Backend for geocaching.com'}, - 'opencaching-com': {'class': OpenCachingComCacheDownloader, 'name': 'opencaching.com', 'description': 'Backend for opencaching.com'}, + #'opencaching-com': {'class': OpenCachingComCacheDownloader, 'name': 'opencaching.com', 'description': 'Backend for opencaching.com'}, } def get(name, *args, **kwargs): @@ -940,17 +1022,15 @@ def get(name, *args, **kwargs): if __name__ == '__main__': import sys - import downloader - import colorer + from advancedcaching import colorer, downloader logger.setLevel(logging.DEBUG) logging.basicConfig(level=logging.DEBUG, format='%(relativeCreated)6d %(levelname)10s %(name)-20s %(message)s', ) parser = GeocachingComCacheDownloader - - + outfile = None - if len(sys.argv) != 3: + if len(sys.argv) != 3: logger.error("Please provide username and password on the command line.") sys.exit(2) @@ -962,7 +1042,7 @@ def get(name, *args, **kwargs): def pcache(c): logger.info("--------------------\nName: '%s'\nTitle: '%s'\nType: %s" % (c.name, c.title, c.type)) - + def dummy_callback(x): return None coords = a.get_overview((geo.Coordinate(49.3513,6.583), geo.Coordinate(49.352,6.584)), dummy_callback) @@ -970,19 +1050,19 @@ def dummy_callback(x): for x in coords: pcache(x) if x.name == 'GC1N8G6': - if x.type != GeocacheCoordinate.TYPE_REGULAR or x.title != 'Druidenpfad': + if x.type != TYPE_REGULAR or x.title != 'Druidenpfad': sys.exit("Wrong type or title (Type is %d, Title is '%s')" % (x.type, x.title)) m = x break - + else: logger.error("Didn't find my own geocache :-(") sys.exit(-1) - + res = a.update_coordinate(m, num_logs = 20) c = res - + errors = 0 if c.owner != 'webhamster': logger.error("Owner doesn't match ('%s', expected webhamster)" % c.owner) @@ -1012,23 +1092,21 @@ def dummy_callback(x): if len(c.attributes) < 20: logger.error("Expected 20 characters of attributes, got %d: '%s'" % (len(c.attributes), c.attributes)) errors += 1 - + link = 'http://wandern-plus.de/saarland/rehlingen-siersburg/weg_2_info.html' if c.websitelink != link: logger.error("Expected website link to be '%s', found '%s'." % (link, c.websitelink)) errors += 1 - + if len(c.get_logs()) < 2: logger.error("Expected at least 2 logs, got %d" % len(c.get_logs())) errors += 1 - + logger.info(u"Owner:%r (type %r)\nTitle:%r (type %r)\nTerrain:%r\nDifficulty:%r\nDescription:%r (type %r)\nShortdesc:%r (type %r)\nHints:%r (type %r)\nLogs: %r\nAttributes: %r" % (c.owner, type(c.owner), c.title, type(c.title), c.get_terrain(), c.get_difficulty(), c.desc[:200], type(c.desc), c.shortdesc, type(c.shortdesc), c.hints, type(c.hints), c.get_logs()[:3], c.attributes)) logger.info(c.get_waypoints()) - + if errors > 0: sys.exit("Found %d error(s)." % errors) - + logger.info("Seems to be okay.") sys.exit(0) - - diff --git a/advancedcaching/cli.py b/advancedcaching/cli.py index 3111353..672e3ff 100644 --- a/advancedcaching/cli.py +++ b/advancedcaching/cli.py @@ -20,12 +20,13 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -import geocaching -import sys -import geo import math import os import re +import sys + +from advancedcaching import geo, geocaching + usage = r'''Here's how to use this app: @@ -52,6 +53,20 @@ Import geocaches, put them into the internal database, filter the imported geocaches and run the actions. %(name)s sql "SELECT * FROM geocaches WHERE ... ORDER BY ... LIMIT ..." do [actions] Select geocaches from local database and run the actions afterwards. Additional use of the filter is also supported. To get more information, run "%(name)s sql". +%(name)s fieldnote [geocache-id] [type] [date] [text] + Write (but don't upload) a fieldnote of type [type] for the geocache defined by [geocache-id] with text [text]. + Date must be given in the following format: YYYY-MM-DD + Available log types: + 0 - just store the text, never upload this note + 1 - log as found + 2 - log as not found + 3 - write note +%(name)s log [geocache-id] [type] [date] [text] + As above, but this will create a log entry on the web page. +%(name)s show-notes + List all stored logs/fieldnotes. +%(name)s upload + Upload stored logs/fieldnotes. options: --user(name) username --pass(word) password @@ -228,10 +243,18 @@ def parse_input (self): self.parse_actions() elif sys.argv[self.nt] == 'update': self.perform_update() + elif sys.argv[self.nt] == 'fieldnote': + self.parse_note(geocaching.GeocacheCoordinate.UPLOAD_AS_FIELDNOTE) + elif sys.argv[self.nt] == 'log': + self.parse_note(geocaching.GeocacheCoordinate.UPLOAD_AS_LOG) + elif sys.argv[self.nt] == 'show-notes': + self.parse_show_notes() + elif sys.argv[self.nt] == 'upload': + self.parse_upload() elif sys.argv[self.nt] == '-v': self.nt += 1 else: - raise ParseError("Expected 'import', 'sql', 'filter' or 'do' but found '%s'" % sys.argv[self.nt], self.nt - 1) + raise ParseError("Expected 'import', 'sql', 'filter', 'do', 'update', 'fieldnote', 'log', 'show-notes', 'upload', but found '%s'" % sys.argv[self.nt], self.nt - 1) self.core.prepare_for_disposal() @@ -252,6 +275,38 @@ def parse_set(self): raise ParseError("I don't understand '%s'" % token) print "* Finished setting options." + def parse_note(self, t): + self.nt += 1 + try: + geocache, logtype, logdate, logtext = sys.argv[self.nt:self.nt+4] + except ValueError, e: + raise ParseError("Expected geocache-id, note type, date and text.") + self.nt += 5 + + c = self.core.get_geocache_by_name(geocache) + + if re.match(r'^\d\d\d\d-\d\d-\d\d$', logdate) == None: + raise ParseError("Expected date in YYYY-MM-DD format, found %s instead." % logdate) + + c.logas = logtype + c.logdate = logdate + c.fieldnotes = unicode(logtext, sys.stdin.encoding) + c.upload_as = t + self.core.save_fieldnote(c) + + def parse_show_notes(self): + self.nt += 1 + l = self.core.pointprovider.get_new_fieldnotes() + print "Geocaches with fieldnotes" + print "-------------------------" + for c in l: + t = "Log" if (geocaching.GeocacheCoordinate.UPLOAD_AS_LOG == c.upload_as) else "Fieldnote" + print '%s: %s (%s) - Type %d - Date %s - Text "%s"' % (t, c.name, c.title, c.logas, c.logdate, c.fieldnotes) + + def parse_upload(self): + self.nt += 1 + self.core.upload_fieldnotes(sync=True) + def parse_import(self): self.nt += 1 diff --git a/advancedcaching/colorer.py b/advancedcaching/colorer.py index 813bb73..3a14bc5 100644 --- a/advancedcaching/colorer.py +++ b/advancedcaching/colorer.py @@ -1,7 +1,9 @@ #!/usr/bin/env python # source: http://stackoverflow.com/questions/384076/how-can-i-make-the-python-logging-output-to-be-colored # encoding: utf-8 + import logging + # now we patch Python code to add color support to logging.StreamHandler def add_coloring_to_emit_windows(fn): # add methods we need to the class @@ -25,7 +27,7 @@ def new(*args): FOREGROUND_RED = 0x0004 # text color contains red. FOREGROUND_INTENSITY = 0x0008 # text color is intensified. FOREGROUND_WHITE = FOREGROUND_BLUE|FOREGROUND_GREEN |FOREGROUND_RED - # winbase.h + # winbase.h STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -99,4 +101,4 @@ def new(*args): logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) else: # all non-Windows platforms are supporting ANSI escapes so we use them - logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) \ No newline at end of file + logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) diff --git a/advancedcaching/connection.py b/advancedcaching/connection.py index 6d43f06..d61a9a1 100644 --- a/advancedcaching/connection.py +++ b/advancedcaching/connection.py @@ -21,6 +21,8 @@ # import logging + + logger = logging.getLogger('connection') offline = False diff --git a/advancedcaching/constants.py b/advancedcaching/constants.py new file mode 100644 index 0000000..becb9ca --- /dev/null +++ b/advancedcaching/constants.py @@ -0,0 +1,47 @@ +TYPE_UNKNOWN = 'unknown' + +TYPE_REGULAR = 'regular' +TYPE_MULTI = 'multi' +TYPE_MYSTERY = 'mystery' +TYPE_LETTERBOX = 'letterbox' +TYPE_WHERIGO = 'wherigo' +TYPE_EVENT = 'event' +TYPE_MEGAEVENT = 'megaevent' +TYPE_TRASHEVENT = 'trashevent' +TYPE_EARTH = 'earth' + +# Grandfathered types of caches +TYPE_VIRTUAL = 'virtual' +TYPE_WEBCAM = 'webcam' + +TYPE_LABELS = { + TYPE_UNKNOWN: 'Unknown Cache', + TYPE_REGULAR: 'Traditional Cache', + TYPE_MULTI: 'Multi-cache', + TYPE_MYSTERY: 'Mystery Cache', + TYPE_LETTERBOX: 'Letterbox Hybrid', + TYPE_WHERIGO: 'Wherigo Cache', + TYPE_EVENT: 'Event Cache', + TYPE_MEGAEVENT: 'Mega-Event Cache', + TYPE_TRASHEVENT: 'Cache In Trash Out Event', + TYPE_EARTH: 'EarthCache', + TYPE_VIRTUAL: 'Virtual Cache', + TYPE_WEBCAM: 'Webcam Cache', +} + +TYPES = TYPE_LABELS.keys() + +# Map name of image at geocaching.com to cache type. +GC_TYPE_MAP = { + 2: TYPE_REGULAR, + 3: TYPE_MULTI, + 8: TYPE_MYSTERY, + 5: TYPE_LETTERBOX, + 1858: TYPE_WHERIGO, + 6: TYPE_EVENT, + 453: TYPE_MEGAEVENT, + 13: TYPE_TRASHEVENT, + 137: TYPE_EARTH, + 4: TYPE_VIRTUAL, + 11: TYPE_WEBCAM, +} diff --git a/advancedcaching/coordfinder.py b/advancedcaching/coordfinder.py index 0d90f4a..c93ef75 100644 --- a/advancedcaching/coordfinder.py +++ b/advancedcaching/coordfinder.py @@ -21,6 +21,13 @@ # from __future__ import division + +import logging +import re + +from advancedcaching import geo + + TEST = (''' @@ -80,9 +87,6 @@
''' -import geo -import re -import logging logger = logging.getLogger('coordfinder') class CalcCoordinateManager(object): @@ -278,7 +282,7 @@ def is_calc_string(text): return (re.match(regex, text) != None) if __name__ == "__main__": - from simplegui import SimpleGui + from advancedcaching.simplegui import SimpleGui print '\n\n=========================================================' h = SimpleGui._strip_html(HTML) #print h diff --git a/advancedcaching/core.py b/advancedcaching/core.py index 269e6ee..c6bc1c7 100755 --- a/advancedcaching/core.py +++ b/advancedcaching/core.py @@ -22,76 +22,72 @@ from __future__ import with_statement -# This is also evaluated by the build scripts -VERSION='0.9.1.2' +import gobject import logging import logging.handlers -logging.basicConfig(level=logging.WARNING, - format='%(relativeCreated)6d %(levelname)10s %(name)-20s %(message)s // %(filename)s:%(lineno)s', - ) - +import os +import threading -from geo import Coordinate +from datetime import datetime +from os import path, mkdir, extsep, remove, walk +from re import sub +from sys import argv, exit, path as sys_path try: from json import loads, dumps except (ImportError, AttributeError): from simplejson import loads, dumps -from sys import argv, exit -from sys import path as sys_path -import downloader -import geocaching -import gpsreader -from os import path, mkdir, extsep, remove, walk -import provider -from threading import Thread -import cachedownloader -from actors.tts import TTS -from re import sub -import threading -from datetime import datetime +# Add parent directory to the path, so we can use advancedcaching in imports +sys_path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from advancedcaching import cachedownloader, connection, downloader, geocaching, gpsreader, provider +from advancedcaching.actors.tts import TTS +from advancedcaching.geo import Coordinate + + +# This is also evaluated by the build scripts +VERSION='0.9.1.2' + + +logging.basicConfig(level=logging.WARNING, + format='%(relativeCreated)6d %(levelname)10s %(name)-20s %(message)s // %(filename)s:%(lineno)s') -import connection -import gobject if len(argv) == 1: - import cli + from advancedcaching import cli print cli.usage % ({'name': argv[0]}) exit() if '-v' in argv or '--remote' in argv: - import colorer + from advancedcaching import colorer logging.getLogger('').setLevel(logging.DEBUG) logging.debug("Set log level to DEBUG") - + if '--debug-http' in argv: downloader.enable_http_debugging() - + extensions = [] if '--simple' in argv: - import simplegui + from advancedcaching import simplegui gui = simplegui.SimpleGui gps = 'gpsdprovider' -elif '--desktop' in argv: - import biggui - gui = biggui.BigGui elif '--qml' in argv: - import qmlgui + from advancedcaching import qmlgui gui = qmlgui.QmlGui gps = 'qmllocationprovider' elif '--hildon' in argv: connection.init() # is only used on the maemo platform - import hildongui + from advancedcaching import hildongui gui = hildongui.HildonGui gps = 'locationgpsprovider' extensions.append('geonames') extensions.append('tts') else: - import cli + from advancedcaching import cli gui = cli.Cli gps = None extensions.append('geonames') - + if '--sim' in argv: gps = 'simulatingprovider' elif '--nogps' in argv: @@ -126,10 +122,10 @@ class Core(gobject.GObject): DATA_DIR = path.expanduser(path.join('~', '')) if not path.exists(MAEMO_HOME) else MAEMO_HOME UPDATE_MODULES = [cachedownloader] - + updating_lock = threading.Lock() _geocache_by_name_event = threading.Event() - + DEFAULT_SETTINGS = { 'download_visible': True, 'download_notfound': True, @@ -172,11 +168,11 @@ class Core(gobject.GObject): 'options_backend': 'geocaching-com-new', 'options_redownload_after': 14, } - + def __init__(self, guitype, gpstype, extensions): """ Initialize the application. - + guitype -- Python type of the gui which is to be used. gpstype -- String indicating the desired GPS access method. extensions -- List of strings indicating desired extensions. @@ -192,26 +188,25 @@ def __init__(self, guitype, gpstype, extensions): self._install_updates() self.__read_config() - + # Check tile URLs for outdated URLs after Openstreetmap URL change for name, details in self.settings['map_providers']: prev = details['remote_url'] details['remote_url'] = sub(r'//(.*).openstreetmap.org/([a-z]*/)?', '//tile.openstreetmap.org/', prev) if prev != details['remote_url']: logger.info("Replaced url '%s' with '%s' because Openstreetmaps changed their URLs." % (prev, details['remote_url'])) - + self.connect('settings-changed', self.__on_settings_changed) self.connect('save-settings', self.__on_save_settings) self.create_recursive(self.settings['download_output_dir']) self.create_recursive(self.settings['download_map_path']) - + self.downloader = downloader.FileDownloader(self.COOKIE_FILE) - + self.pointprovider = provider.PointProvider(self.CACHES_DB, geocaching.GeocacheCoordinate) self.gui = guitype(self) - - + if ('debug_log_to_http' in self.settings and self.settings['debug_log_to_http']) or '--remote' in argv: http_handler = logging.handlers.HTTPHandler("danielfett.de", "http://www.danielfett.de/files/collect.php") buffering_handler = logging.handlers.MemoryHandler(100, target = http_handler) @@ -220,19 +215,18 @@ def __init__(self, guitype, gpstype, extensions): logging.debug("Remote logging activated!") # Now reset the setting to default self.settings['debug_log_to_http'] = False - - + self.emit('settings-changed', self.settings, self) - self.emit('fieldnotes-changed') + self.emit('fieldnotes-changed') self.__setup_gps(gps) - + if 'geonames' in extensions: - import geonames + from advancedcaching import geonames self.geonames = geonames.Geonames(self.downloader) - + if 'tts' in extensions: - from actors.notify import Notify + from advancedcaching.actors.notify import Notify actor_tts = TTS(self) actor_tts.connect('error', lambda caller, msg: self.emit('error', msg)) actor_notify = Notify(self) @@ -254,7 +248,7 @@ def __init__(self, guitype, gpstype, extensions): def create_recursive(dpath): """ Create dpath and all parent directories if necessary - + """ if dpath != '/': if not path.exists(dpath): @@ -269,7 +263,7 @@ def create_recursive(dpath): def optimize_data(self): """ Clean up database and file system. - + Removes found geocaches and their images from the database or filesystem, respectively. """ self.pointprovider.push_filter() @@ -292,7 +286,7 @@ def optimize_data(self): def get_file_sizes(self): """ Return the accumulated size of the images in the download output directory (i.e., the images). - + """ folder = self.settings['download_output_dir'] folder_size = 0 @@ -307,7 +301,7 @@ def get_file_sizes(self): def format_file_size(size): """ Format a file size to a human readable string. - + """ if size < 1024: return "%d B" % size @@ -317,7 +311,6 @@ def format_file_size(size): return "%d MiB" % (size / (1024 * 1024)) else: return "%d GiB" % (size / (1024 * 1024 * 1024)) - ############################################## # @@ -327,8 +320,8 @@ def format_file_size(size): def _install_updates(self): """ - Installs updated modules. - + Installs updated modules. + Checks the updates directory for new versions of one of the modules listed in self.UPDATE_MODULES and reloads the modules. Version check is performed by comparing the VERSION variable stored in the module. """ updated_modules = 0 @@ -344,7 +337,7 @@ def _install_updates(self): exec line in v_dict break else: - logger.error("Could not find VERSION string in file %s!" % modulefile) + logger.error("Could not find VERSION string in file %s!" % modulefile) continue if v_dict['VERSION'] > m.VERSION: logging.info("Reloading Module '%s', current version number: %d, new version number: %d" % (m.__name__, v_dict['VERSION'], m.VERSION)) @@ -361,7 +354,7 @@ def _install_updates(self): def try_update(self, silent = False): """ Retrieve and install updates. - + This method connects to danielfett.de and tries to retrieve an md5sums file containing references to updated files for this AGTL version. If available, downloads the files and checks their md5sums before copying them to the updates folder. Calls self._install_updates afterwards to reload updated modules. silent -- If possible, suppress errors. Useful for auto-updating. @@ -370,23 +363,23 @@ def try_update(self, silent = False): if not silent: self.emit('error', Exception("Can't update in offline mode.")) return False - + class NoUpdateException(Exception): pass - - - from urllib import urlretrieve - from urllib2 import HTTPError - import tempfile + import hashlib + import tempfile from shutil import copyfile + from urllib import urlretrieve + from urllib2 import HTTPError + self.create_recursive(self.UPDATE_DIR) baseurl = 'http://www.danielfett.de/files/agtl-updates/%s' % VERSION url = "%s/updates" % baseurl self.emit('progress', 0.5, "Checking for updates...") try: try: - reader = self.downloader.get_reader(url, login=False) + reader = self.downloader.get_reader(url) except HTTPError, e: raise NoUpdateException("No updates available.") except Exception, e: @@ -430,7 +423,7 @@ class NoUpdateException(Exception): remove(temp) except Exception: pass - + except NoUpdateException, e: self.emit('hide-progress') return self._install_updates() @@ -452,13 +445,13 @@ class NoUpdateException(Exception): def save_settings(self, settings, source): ''' This should be called to update the settings throughout all components. - + When settings need to be changed, for example when the GUI updates the username, other components need to be notified. This method updates the settings and notifies the other components (including the core). settings -- Subset of settings dictionary which is to be updated. source -- Calling class instance, for example a HildonGui instance. This is passed on so that the triggering component can suppress reactions to its own updates. ''' logger.debug("Got settings update from %s" % source) - + self.settings.update(settings) self.emit('settings-changed', settings, source) @@ -466,15 +459,15 @@ def save_settings(self, settings, source): def __on_settings_changed(self, caller, settings, source): ''' This is called when settings have changed. - + This method is connected to the settings-changed signal and indicates that someone has changed the settings. The new settings need to be applied, e.g., to the cachedownloader. settings -- contains only the updated parts of the settings dictionary. ''' logger.debug("Settings where changed by %s." % source) - + if 'options_backend' in settings or 'download_output_dir' in settings or 'download_noimages' in settings: self.__install_cachedownloader() - + if source == self: return if 'options_username' in settings: @@ -485,7 +478,7 @@ def __on_settings_changed(self, caller, settings, source): def __on_save_settings(self, caller): """ This is called when settings shall be saved, calling save_settings afterwards. - + The save-settings signal is emitted whenever the application is about to be destroyed. Therefore, we need to save some settings. """ logger.debug("Assembling update for settings, on behalf of %s" % caller) @@ -495,7 +488,7 @@ def __on_save_settings(self, caller): 'last_target_lon': self.current_target.lon } caller.save_settings(settings, self) - + def __del__(self): logger.debug("Somebody is trying to kill me, saving the settings.") self.emit('save-settings') @@ -504,7 +497,7 @@ def __del__(self): def prepare_for_disposal(self): """ This is called by the GUI when it is about to be killed. - + """ logger.debug("Somebody is being killed, saving the settings.") self.emit('save-settings') @@ -515,7 +508,7 @@ def __read_config(self): filename = path.join(self.SETTINGS_DIR, 'config') logger.debug("Loading config from %s" % filename) if not path.exists(filename): - logger.error("Did not find settings file (%s), loading default settings." % filename) + logger.warning("Did not find settings file (%s), loading default settings." % filename) self.settings = self.DEFAULT_SETTINGS return with file(filename, 'r') as f: @@ -551,6 +544,8 @@ def __install_cachedownloader(self): self.cachedownloader.disconnect(handler) self.__cachedownloader_signal_handlers = [] self.cachedownloader = cachedownloader.get(self.settings['options_backend'], self.downloader, self.settings['download_output_dir'], not self.settings['download_noimages']) + self.cachedownloader.update_userdata(username = self.settings['options_username']) + self.cachedownloader.update_userdata(password = self.settings['options_password']) a = self.cachedownloader.connect("download-error", self.on_download_error) b = self.cachedownloader.connect("already-downloading-error", self.on_already_downloading_error) c = self.cachedownloader.connect('progress', self.on_download_progress) @@ -566,7 +561,7 @@ def __install_cachedownloader(self): def set_target(self, coordinate): """ Sets the new target coordinate. - + Updates target distance and target bearing afterwards and emits target-changed signal. """ self.current_target = coordinate @@ -581,15 +576,15 @@ def __get_target_distance_bearing(self): distance = None bearing = None return distance, bearing - + def __setup_gps(self, gps): """ Setup GPS provider according to the constant in gps. - + """ if gps == 'simulatingprovider': self.gps_thread = gpsreader.FakeGpsReader(self) - gobject.timeout_add(1000, self.__read_gps) + gobject.timeout_add(1000, lambda: self.__read_gps(self.gps_thread.get_data())) self.set_target(gpsreader.FakeGpsReader.get_target()) elif gps == None: self.gps_thread = gpsreader.FakeGpsReader(self) @@ -603,11 +598,11 @@ def __setup_gps(self, gps): elif gps == 'qmllocationprovider': self.gui.get_gps(self.__read_gps) self.gps_thread = None - + def __read_gps(self, fix): """ This callback method is called by the gpsreader to process a new fix. - + """ if fix.position != None: self.current_position = fix.position @@ -625,7 +620,7 @@ def __read_gps(self, fix): # Geonames & Routing # ############################################## - + def get_coord_by_name(self, query): return self.geonames.search(query) @@ -642,10 +637,10 @@ def get_route(self, c1, c2, r): MAX_TOGETHER = 20 for i in range(len(route)): if len(together) == 0: - together = [route[i]] + together = [route[i]] if (i < len(route) - 1): brg = route[i].bearing_to(route[i + 1]) - + if len(together) < MAX_TOGETHER \ and (i < len(route) - 1) \ and (abs(brg - 90) < TOL @@ -675,23 +670,23 @@ def get_route(self, c1, c2, r): # Filters, Searching & Pointprovider # ############################################## - + def set_filter(self, found=None, owner_search='', name_search='', size=None, terrain=None, diff=None, ctype=None, location=None, marked=None): """ - Sets a new filter for the pointprovider. - + Sets a new filter for the pointprovider. + Is mainly used to filter the map display. (Currently only in hildongui_plugins) - + """ self.pointprovider.set_filter(found=found, owner_search=owner_search, name_search=name_search, size=size, terrain=terrain, diff=diff, ctype=ctype, marked=marked) self.emit('map-marks-changed') - + def reset_filter(self): """ - Resets the filter for the pointprovider. - + Resets the filter for the pointprovider. + (Currently only in hildongui_plugins) - + """ self.pointprovider.set_filter() self.emit('map-marks-changed') @@ -700,7 +695,7 @@ def get_points_filter(self, found=None, owner_search='', name_search='', size=No """ Performs a search according to the given criteria and returns the geocaches. Also returns information on whether the result was truncated due to the maximum number of search results configured in pointprovider. preserve_filter -- Apply the filter and keep it active after this method. If set to False, the filtering remains unchanged after the method call. - + """ if not preserve_filter: self.pointprovider.push_filter() @@ -715,7 +710,7 @@ def get_points_filter(self, found=None, owner_search='', name_search='', size=No def get_geocache_by_name(self, name): """ Return a geocache by its ID. - + """ return self.pointprovider.get_by_name(name) @@ -724,7 +719,7 @@ def get_geocache_by_name(self, name): # Downloading # ############################################## - + def _download_upload_helper(self, action, then, *args, **kwargs): with Core.updating_lock: self._check_auto_update() @@ -744,13 +739,13 @@ def _check_auto_update(self): def download_overview(self, location, sync=False, skip_callback = None): """ Downloads an *overview* of geocaches within the boundaries given in location. - + location -- Geographic boundaries (see cachedownloader.get_overview for details) sync -- Perform actions synchronized, i.e., don't use threads. skip_callback -- A callback function which gets the geocache id and its found status as input. If it returns true, the geocache's details are not downloaded. """ - if not sync: - t = Thread(target=self._download_upload_helper, args=['self.cachedownloader.get_overview', self._download_overview_complete, location, self.get_geocache_by_name_async, skip_callback]) + if not sync: + t = threading.Thread(target=self._download_upload_helper, args=['self.cachedownloader.get_overview', self._download_overview_complete, location, self.get_geocache_by_name_async, skip_callback]) t.daemon = True t.start() return False @@ -760,7 +755,7 @@ def download_overview(self, location, sync=False, skip_callback = None): def _download_overview_complete(self, caches, sync=False): """ Called upon completion of the download of all geocaches within a boundary. - + caches -- Updated geocache information. sync -- Perform actions synchronized, i.e., don't use threads. """ @@ -771,10 +766,10 @@ def _download_overview_complete(self, caches, sync=False): if point_new: new_caches.append(c) self.pointprovider.save() - + for c in caches: self.emit('cache-changed', c) - + self.emit('hide-progress') self.emit('map-marks-changed') if sync: @@ -785,13 +780,13 @@ def _download_overview_complete(self, caches, sync=False): def download_cache_details(self, cache, sync=False): """ Download or update *detailed* information for a specific geocache. - + location -- Geographic boundaries (see cachedownloader.get_overview for details) sync -- Perform actions synchronized, i.e., don't use threads. - + """ - if not sync: - t = Thread(target=self._download_upload_helper, args=['self.cachedownloader.update_coordinate', self._download_cache_details_complete, cache, self.settings['download_num_logs']]) + if not sync: + t = threading.Thread(target=self._download_upload_helper, args=['self.cachedownloader.update_coordinate', self._download_cache_details_complete, cache, self.settings['download_num_logs']]) t.daemon = True t.start() #t.join() @@ -818,12 +813,11 @@ def _download_cache_details_complete(self, cache, sync = False): def download_cache_details_map(self, location, visibleonly=False): """ Download *details* for all geocaches within a specific location. - + location -- Geographic boundaries (see cachedownloader.get_overview for details) sync -- Perform actions synchronized, i.e., don't use threads. - + """ - self.pointprovider.push_filter() if self.settings['download_notfound'] or visibleonly: @@ -852,24 +846,24 @@ def download_cache_details_map(self, location, visibleonly=False): def download_cache_details_list(self, caches, sync=False): """ Download/update *detailed* information for a list of geocaches. - + caches -- List of geocaches - + """ if not sync: - t = Thread(target=self._download_upload_helper, args=['self.cachedownloader.update_coordinates', self._download_cache_details_list_complete, caches, self.settings['download_num_logs']]) + t = thrading.Thread(target=self._download_upload_helper, args=['self.cachedownloader.update_coordinates', self._download_cache_details_list_complete, caches, self.settings['download_num_logs']]) t.daemon = True t.start() return False else: return self._download_cache_details_list_complete(self.cachedownloader.update_coordinates(caches)) - + def _download_cache_details_list_complete(self, caches): """ Called when details for a list of geocaches were downloaded. - + caches -- List of geocaches - + """ for c in caches: self.pointprovider.add_point(c, True) @@ -883,12 +877,12 @@ def _download_cache_details_list_complete(self, caches): def on_download_progress(self, something, text, i, max_i): """ Signal handler which is called when the downloading process has made progress. - + something -- not used text -- Text describing the current work item i -- Progress in relation to max_i max_i -- Expected maximum value of i (Actual displayed progress fraction is i/max_i) - + """ logger.debug("Progress: %f of %f" % (i, max_i)) self.emit('progress', float(i) / float(max_i), "%s..." % text) @@ -897,45 +891,44 @@ def on_download_progress(self, something, text, i, max_i): def on_already_downloading_error(self, something, error): """ Signal handler which is called when the downloading thread cannot download because someone else is still downloading. - + """ self.emit('error', error) - + def on_download_error(self, something, error): """ Signal handler which is called when an error happened during a download (or upload). - + """ extra_message = "Error:\n%s" % error logging.exception(error) self.emit('hide-progress') self.emit('error', extra_message) - + def default_download_skip_callback(self, geocache, found): """ - This is a default callback for skip_callback in download_overview. - + This is a default callback for skip_callback in download_overview. + This callback is called after the cachedownloader fetched a list of geocaches which are in a certain area and before it actually downloads the geocache's details. If the callback returns True, the cachedownloader will skip downloading details. It reads the settings and acts accordingly. - + geocache -- is None or the geocache which was fetched from the database before its details were updated found -- the (new) found status, as read from the web page - """ if self.settings['download_notfound'] and found: logger.debug("Geocache is marked as found, skipping!") return True - if geocache == None or geocache.get_updated() == None: + if geocache == None or geocache.get_updated() == None: logger.debug("Geocache %r was not in the database or had no update timestamp." % geocache) return False # When the geocache is not in the database or when it was downloaded before we introduced timestamps, don't skip! if self.settings['options_redownload_after'] > 0: - diff = datetime.now() - geocache.get_updated() + diff = datetime.now() - geocache.get_updated() if diff.days >= self.settings['options_redownload_after']: logger.debug("Geocache %s was not updated for %d days. It's time!" % (geocache.name, diff.days)) return False logger.debug("Geocache %s was not updated for %d days. That's fine." % (geocache.name, diff.days)) return True return False - + def get_geocache_by_name_async(self, id): self._geocache_by_name_event.clear() gobject.idle_add(self._get_geocache_by_name_async_idle, id, priority=gobject.PRIORITY_HIGH) @@ -944,7 +937,7 @@ def get_geocache_by_name_async(self, id): logger.error("Screw it!") return None return self._geocache_by_name_result - + def _get_geocache_by_name_async_idle(self, id): self._geocache_by_name_result = self.get_geocache_by_name(id) self._geocache_by_name_event.set() @@ -954,13 +947,13 @@ def _get_geocache_by_name_async_idle(self, id): # Exporting # ############################################## - + def export_cache(self, cache, format, folder): """ Export descriptions of geocaches. Not maintained at the moment. - + """ - from exporter import GpxExporter + from advancedcaching.exporter import GpxExporter if (format == 'gpx'): exporter = GpxExporter() else: @@ -973,7 +966,6 @@ def export_cache(self, cache, format, folder): self.emit('error', e) finally: self.emit('hide-progress') - ############################################## # @@ -984,7 +976,7 @@ def export_cache(self, cache, format, folder): def save_fieldnote(self, cache): """ Save the fieldnote information for a geocache to the database. - + """ if cache.logas == geocaching.GeocacheCoordinate.LOG_AS_FOUND: cache.found = 1 @@ -992,24 +984,28 @@ def save_fieldnote(self, cache): elif cache.logas == geocaching.GeocacheCoordinate.LOG_AS_NOTFOUND: cache.found = 0 - self.save_cache_attribute(cache, ('logas', 'logdate', 'fieldnotes', 'found')) - self.emit('fieldnotes-changed') + self.save_cache_attribute(cache, ('logas', 'logdate', 'fieldnotes', 'found', 'upload_as')) + self.emit('fieldnotes-changed') + self.emit('cache-changed', cache) - def upload_fieldnotes(self): + def upload_fieldnotes(self, sync = False): """ - Upload fieldnotes to the web site. - + Upload fieldnotes and logs to the web site. + """ caches = self.pointprovider.get_new_fieldnotes() - - t = Thread(target=self._download_upload_helper, args=['self.cachedownloader.upload_fieldnotes', self._upload_fieldnotes_complete, caches]) - t.daemon = True - t.start() - + if not sync: + t = threading.Thread(target=self._download_upload_helper, args=['self.cachedownloader.upload_fieldnotes_and_logs', self._upload_fieldnotes_complete, caches]) + t.daemon = True + t.start() + else: + res = self.cachedownloader.upload_fieldnotes_and_logs(caches) + self._upload_fieldnotes_complete(res) + def _upload_fieldnotes_complete(self, caches): """ - Called when uploading of fieldnotes is complete. - + Called when uploading of fieldnotes and logs is complete. + Resets the fieldnotes which were uploaded to "NO LOG". """ for c in caches: @@ -1022,7 +1018,7 @@ def _upload_fieldnotes_complete(self, caches): def get_new_fieldnotes_count(self): """ Return the number of pending fieldnotes. - + """ return self.pointprovider.get_new_fieldnotes_count() @@ -1036,7 +1032,7 @@ def get_new_fieldnotes_count(self): def save_cache_attribute(self, cache, attribute): """ Save the attribute of a geocache to the database. - + """ if type(attribute) == tuple: for a in attribute: @@ -1048,7 +1044,7 @@ def save_cache_attribute(self, cache, attribute): def set_alternative_position(self, cache, ap): """ Sets the alternative position for a geocaches. - + An alternative position is a user-chosen geographic location for a geocache which differs from the original location. This is, for example, used for solved mystery geocaches. """ cache.set_alternative_position(ap) @@ -1059,7 +1055,7 @@ def set_alternative_position(self, cache, ap): def start(): """ Start the application. - + """ gobject.threads_init() Core(gui, gps, extensions) @@ -1067,7 +1063,7 @@ def start(): def start_profile(what): """ Uses cprofile to profile the method calls in the application. For developing only. - + """ import cProfile p = cProfile.Profile() @@ -1090,16 +1086,16 @@ def c(x, y): for line in line.calls[:10]: print "-- %d %4f %s" % (line.callcount, line.totaltime, line.code) - + print "BY TOTALTIME:\n------------------------------------------------------------" - def c(x, y): + def c2(x, y): if x.totaltime < y.totaltime: return 1 elif x.totaltime == y.totaltime: return 0 else: return -1 - stats.sort(cmp=c) + stats.sort(cmp=c2) for line in stats[:30]: print "%d %4f %s" % (line.callcount, line.totaltime, line.code) if line.calls == None: @@ -1113,4 +1109,3 @@ def c(x, y): start_profile('start()') else: start() - diff --git a/advancedcaching/downloader.py b/advancedcaching/downloader.py index 1c9d80a..49b457b 100644 --- a/advancedcaching/downloader.py +++ b/advancedcaching/downloader.py @@ -20,16 +20,25 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # - from __future__ import with_statement + import logging -logger = logging.getLogger('downloader') -import connection +import socket +from cookielib import LWPCookieJar +from httplib import HTTPSConnection from sys import argv -from urllib2 import build_opener, install_opener, HTTPCookieProcessor, HTTPHandler -from urllib2 import Request, urlopen from urllib import urlencode -from cookielib import LWPCookieJar +from urllib2 import build_opener, install_opener, HTTPCookieProcessor, HTTPHandler, HTTPSHandler, Request, urlopen + +from advancedcaching import connection + +try: + import ssl +except ImportError: + ssl = None + + +logger = logging.getLogger('downloader') DEBUG_HTTP = False @@ -40,7 +49,27 @@ def enable_http_debugging(): DEBUG_PATH = '' DEBUG_COUNTER = 0 logger.info("Writing debug HTTP logs.") - + + +# SSLv3 classes that bypass bug in openssl library when connecting to geocaching.com ugins SSLv23 protocol. +class SSLv3HTTPSConnection(HTTPSConnection): + """ + Connect using SSLv3. + """ + def connect(self): + "Connect to a host on a given (SSL) port." + + sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3) + + +class SSLv3HTTPSHandler(HTTPSHandler): + def https_open(self, req): + return self.do_open(SSLv3HTTPSConnection, req) + class FileDownloader(): USER_AGENT = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/12.04 Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/535.19' @@ -52,19 +81,19 @@ def __init__(self, cookiefile, core = None): from socket import setdefaulttimeout setdefaulttimeout(30) self.opener_installed = False - + # This controls the use of the cache-headers in requests to allow/deny minified answers # as provided by some mobile operators. self.allow_minified_answers = True - - + self.cj = LWPCookieJar(self.cookiefile) - + handlers = [HTTPCookieProcessor(self.cj)] if DEBUG_HTTP: - opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj)) - else: - opener = build_opener(HTTPCookieProcessor(self.cj)) + handlers.append(HTTPHandler(debuglevel=1)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + handlers.append(SSLv3HTTPSHandler()) + opener = build_opener(*handlers) install_opener(opener) try: @@ -72,14 +101,11 @@ def __init__(self, cookiefile, core = None): logger.info("Loaded cookie file") except IOError, e: logger.info("Couldn't load cookie file") - + if core != None: core.connect('save-settings', self._on_save_settings) - - #todo : username and password need to go to cachedownloader - #todo: - - def _on_save_settings(settings, source): + + def _on_save_settings(self, settings, source): try: self.cj.save() except Exception, e: diff --git a/advancedcaching/exporter.py b/advancedcaching/exporter.py index 6f29727..52372c2 100644 --- a/advancedcaching/exporter.py +++ b/advancedcaching/exporter.py @@ -20,10 +20,13 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -from pyfo import pyfo import os from datetime import datetime -from geocaching import GeocacheCoordinate + +from advancedcaching.geocaching import GeocacheCoordinate +from advancedcaching.pyfo import pyfo + + class Exporter(): def export(self, coordinate, folder = None): diff --git a/advancedcaching/extListview.py b/advancedcaching/extListview.py index 420cb52..1f139e4 100644 --- a/advancedcaching/extListview.py +++ b/advancedcaching/extListview.py @@ -479,7 +479,7 @@ def replaceContent(self, rows): def shuffle(self): - import random + import random """ Shuffle the content of the list """ order = xrange(len(self.store)) random.shuffle(order) diff --git a/advancedcaching/geo.py b/advancedcaching/geo.py index 8105c86..f25aedb 100644 --- a/advancedcaching/geo.py +++ b/advancedcaching/geo.py @@ -22,6 +22,7 @@ import math import re + try: from location import distance_between def distance_to_liblocation(src, target): diff --git a/advancedcaching/geocaching.py b/advancedcaching/geocaching.py index be8203d..721be8e 100644 --- a/advancedcaching/geocaching.py +++ b/advancedcaching/geocaching.py @@ -20,42 +20,29 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # +import logging +import time +from datetime import datetime try: from simplejson import dumps, loads except (ImportError, AttributeError): from json import loads, dumps -from datetime import datetime -import logging -import time +from advancedcaching import geo +from advancedcaching.constants import TYPE_LABELS, TYPE_UNKNOWN + -import geo logger = logging.getLogger('geocaching') + class GeocacheCoordinate(geo.Coordinate): LOG_NO_LOG = 0 LOG_AS_FOUND = 1 LOG_AS_NOTFOUND = 2 LOG_AS_NOTE = 3 - TYPE_REGULAR = 'regular' - TYPE_MULTI = 'multi' - TYPE_VIRTUAL = 'virtual' - TYPE_EVENT = 'event' - TYPE_MYSTERY = 'mystery' - TYPE_WEBCAM = 'webcam' - TYPE_UNKNOWN = 'unknown' - TYPE_EARTH = 'earth' - TYPES = [ - TYPE_REGULAR, - TYPE_MULTI, - TYPE_VIRTUAL, - TYPE_EVENT, - TYPE_MYSTERY, - TYPE_WEBCAM, - TYPE_UNKNOWN, - TYPE_EARTH - ] + UPLOAD_AS_FIELDNOTE = 0 + UPLOAD_AS_LOG = 1 STATUS_NORMAL = 0 STATUS_DISABLED = 1 @@ -77,16 +64,6 @@ class GeocacheCoordinate(geo.Coordinate): SIZES = ['other', 'micro', 'small', 'regular', 'big', 'other'] - TYPE_MAPPING = { - TYPE_MULTI: 'Multi-cache', - TYPE_REGULAR: 'Traditional Cache', - TYPE_EARTH: 'Earthcache', - TYPE_UNKNOWN: 'Unknown Cache', - TYPE_EVENT: 'Event Cache', - TYPE_WEBCAM: 'Webcam Cache', - TYPE_VIRTUAL: 'Virtual Cache' - } - USER_TYPE_COORDINATE = 0 USER_TYPE_CALC_STRING = 1 USER_TYPE_CALC_STRING_OVERRIDE = 2 @@ -95,7 +72,8 @@ class GeocacheCoordinate(geo.Coordinate): 'size', 'difficulty', 'terrain', 'owner', 'found', 'waypoints', \ 'images', 'notes', 'fieldnotes', 'logas', 'logdate', 'marked', \ 'logs', 'status', 'vars', 'alter_lat', 'alter_lon', 'updated', \ - 'user_coordinates', 'attributes', 'last_viewed', 'websitelink') + 'user_coordinates', 'attributes', 'last_viewed', 'websitelink', \ + 'upload_as') # These are the table fields which can safely be updated when # the geocache is re-downloaded. User data should not be contained @@ -135,7 +113,9 @@ class GeocacheCoordinate(geo.Coordinate): 'attributes' : 'TEXT', 'last_viewed' : 'INTEGER', # SQLite doesn't have real DATETIME data type 'websitelink' : 'TEXT', + 'upload_as' : 'INTEGER', } + def __init__(self, lat, lon=None, name='', data=None): geo.Coordinate.__init__(self, lat, lon, name) if data != None: @@ -171,6 +151,7 @@ def __init__(self, lat, lon=None, name='', data=None): self.attributes = '' self.last_viewed = 0 self.websitelink = '' + self.upload_as = self.UPLOAD_AS_FIELDNOTE def clone(self): n = GeocacheCoordinate(self.lat, self.lon) @@ -325,10 +306,10 @@ def add_attribute(self, attr): self.attributes = ','.join(s) def get_gs_type(self): - if self.TYPE_MAPPING.has_key(self.type): - return self.TYPE_MAPPING[self.type] + if self.type in TYPE_LABELS: + return TYPE_LABELS[self.type] else: - return self.TYPE_MAPPING[self.TYPE_UNKNOWN] + return TYPE_LABELS[TYPE_UNKNOWN] def set_alternative_position(self, coord): self.alter_lat = coord.lat @@ -337,7 +318,7 @@ def set_alternative_position(self, coord): def start_calc(self, stripped_desc = None): if stripped_desc == None: stripped_desc = self.desc - from coordfinder import CalcCoordinateManager + from advancedcaching.coordfinder import CalcCoordinateManager if self.vars == None or self.vars == '': vars = {} else: @@ -470,5 +451,3 @@ def get_collected_coordinates(self, format, include_unknown = True, htmlcallback i += 1 logger.debug("Added coordinate, name=%r, title=%r, user_coordinate_id=%r" % (coord.name, coord.title, coord.user_coordinate_id)) return clist - - diff --git a/advancedcaching/geonames.py b/advancedcaching/geonames.py index 8bd0748..c8ff675 100644 --- a/advancedcaching/geonames.py +++ b/advancedcaching/geonames.py @@ -20,18 +20,20 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # +import logging from urllib import quote -import geo try: import json json.dumps except (ImportError, AttributeError): import simplejson as json -import logging +from advancedcaching import geo + logger = logging.getLogger('geonames') + class Geonames(): URL = '''http://ws.geonames.org/searchJSON?formatted=true&q=%(query)s&maxRows=%(max_rows)d&style=short''' URL_STREETS = 'http://ws.geonames.org/findNearestIntersectionOSMJSON?formatted=true&lat=%f&lng=%f&style=short' diff --git a/advancedcaching/gpsreader.py b/advancedcaching/gpsreader.py index acae64f..b5f645b 100644 --- a/advancedcaching/gpsreader.py +++ b/advancedcaching/gpsreader.py @@ -20,18 +20,20 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -import geo -from socket import socket, AF_INET, SOCK_STREAM -from datetime import datetime import logging +from datetime import datetime +from socket import socket, AF_INET, SOCK_STREAM -logger = logging.getLogger('gpsreader') +from advancedcaching import geo try: import location except (ImportError): pass +logger = logging.getLogger('gpsreader') + + class Fix(): BEARING_HOLD_EPD = 90 # arbitrary, yet non-random value last_bearing = 0 diff --git a/advancedcaching/gtkmap.py b/advancedcaching/gtkmap.py index 8a334a2..71cec16 100644 --- a/advancedcaching/gtkmap.py +++ b/advancedcaching/gtkmap.py @@ -20,35 +20,30 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -import logging -import math - -from abstractmap import AbstractGeocacheLayer -from abstractmap import AbstractMap -from abstractmap import AbstractMapLayer -from abstractmap import AbstractMarksLayer import cairo -import geo -import geocaching import gobject import gtk -import openstreetmap +import logging +import math import pango -import threadpool -logger = logging.getLogger('gtkmap') -from os.path import getsize + from hashlib import md5 +from os.path import getsize +from advancedcaching import geo, geocaching, openstreetmap, threadpool +from advancedcaching.abstractmap import AbstractGeocacheLayer, AbstractMap, AbstractMapLayer, AbstractMarksLayer +from advancedcaching.constants import TYPE_REGULAR, TYPE_MULTI -class Map(gtk.DrawingArea, AbstractMap): + +logger = logging.getLogger('gtkmap') +class Map(gtk.DrawingArea, AbstractMap): MIN_DRAG_REDRAW_DISTANCE = 5 DRAG_RECHECK_SPEED = 20 - + LAZY_SET_CENTER_DIFFERENCE = 0.1 # * screen (width|height) - def __init__(self, center, zoom, tile_loader=None, draggable=True): gtk.DrawingArea.__init__(self) AbstractMap.__init__(self, center, zoom, tile_loader) @@ -58,7 +53,7 @@ def __init__(self, center, zoom, tile_loader=None, draggable=True): self.connect("button_press_event", self.__drag_start) self.connect("scroll_event", self.__scroll) self.connect("button_release_event", self.__drag_end) - + if draggable: self.connect("motion_notify_event", self.__drag) self.set_events(gtk.gdk.EXPOSURE_MASK | gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK | gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.SCROLL) @@ -548,9 +543,9 @@ def draw(self): radius = default_radius if c.found: color = found - elif c.type == geocaching.GeocacheCoordinate.TYPE_REGULAR: + elif c.type == TYPE_REGULAR: color = regular - elif c.type == geocaching.GeocacheCoordinate.TYPE_MULTI: + elif c.type == TYPE_MULTI: color = multi else: color = default diff --git a/advancedcaching/hildon_plugins.py b/advancedcaching/hildon_plugins.py index b265f2a..211df0f 100644 --- a/advancedcaching/hildon_plugins.py +++ b/advancedcaching/hildon_plugins.py @@ -20,18 +20,20 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # -import geocaching import gtk import hildon -import pango -import threadpool import logging -import geo -from utils import HTMLManipulations +import pango + +from advancedcaching import geo, geocaching, threadpool +from advancedcaching.constants import TYPE_UNKNOWN, TYPES +from advancedcaching.utils import HTMLManipulations + + logger = logging.getLogger('plugins') + class HildonSearchPlace(object): - def plugin_init(self): self.last_searched_text = '' logger.info("Using Search Place plugin") @@ -355,18 +357,8 @@ def _on_show_search(self, widget, data): if sizes == [1, 2, 3, 4, 5]: sizes = None - typelist = [ - geocaching.GeocacheCoordinate.TYPE_REGULAR, - geocaching.GeocacheCoordinate.TYPE_MULTI, - geocaching.GeocacheCoordinate.TYPE_VIRTUAL, - geocaching.GeocacheCoordinate.TYPE_EARTH, - geocaching.GeocacheCoordinate.TYPE_EVENT, - geocaching.GeocacheCoordinate.TYPE_MYSTERY, - geocaching.GeocacheCoordinate.TYPE_UNKNOWN - ] - - types = [typelist[x] for x, in sel_type.get_selected_rows(0)] - if geocaching.GeocacheCoordinate.TYPE_UNKNOWN in types: + types = [TYPES[x] for x, in sel_type.get_selected_rows(0)] + if TYPE_UNKNOWN in types: types = None # found, marked @@ -586,7 +578,7 @@ def _on_show_about(self, widget, data): text = "%s\n\n%s\n\n" % (copyright, additional) l = gtk.Label('') - import core + from advancedcaching import core l.set_markup("AGTL version %s" % core.VERSION) l.set_alignment(0, 0) page.pack_start(l, False) @@ -606,7 +598,7 @@ def _on_show_about(self, widget, data): notebook.append_page(page, gtk.Label('Update')) l = gtk.Label('') - import cachedownloader + from advancedcaching import cachedownloader l.set_markup("Website parser version %d (from %s)\n\nIf you're having trouble downloading geocaches or uploading fieldnotes, try clicking 'update' to fetch the latest website parser.\n\nAlso check the regular maemo updates from time to time." % (cachedownloader.VERSION, cachedownloader.VERSION_DATE)) l.set_alignment(0, 0) l.set_line_wrap(True) @@ -852,7 +844,6 @@ def _show_tool_rot13(self, caller, data = None): dialog.vbox.pack_start(destination) def do_rot(widget): - import cachedownloader try: text = HTMLManipulations._rot13(source.get_buffer().get_text(source.get_buffer().get_start_iter(), source.get_buffer().get_end_iter())) except Exception: diff --git a/advancedcaching/hildongui.py b/advancedcaching/hildongui.py index 948ff72..b8caa54 100644 --- a/advancedcaching/hildongui.py +++ b/advancedcaching/hildongui.py @@ -20,42 +20,36 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # - + # deps: python-html python-image python-netclient python-misc python-pygtk python-mime python-json - -### For the gui :-) -from math import ceil -from os import extsep -from os import system -from os.path import join, exists -import re +### For the gui :-) -from astral import Astral -import geo -import geocaching import gobject import gtk import hildon -from utils import HTMLManipulations -from hildon_plugins import HildonFieldnotes -from hildon_plugins import HildonSearchPlace -from hildon_plugins import HildonSearchGeocaches -from hildon_plugins import HildonAboutDialog -from hildon_plugins import HildonDownloadMap -from hildon_plugins import HildonToolsDialog +import logging import pango -from portrait import FremantleRotation -from simplegui import SimpleGui -from simplegui import UpdownRows +import re +from math import ceil +from os import extsep, system +from os.path import join, exists from xml.sax.saxutils import escape as my_gtk_label_escape -from gtkmap import Map, OsdLayer, SingleMarkLayer -from coordfinder import CalcCoordinate + +from advancedcaching import geo, geocaching +from advancedcaching.astral import Astral +from advancedcaching.coordfinder import CalcCoordinate +from advancedcaching.gtkmap import Map, OsdLayer, SingleMarkLayer +from advancedcaching.hildon_plugins import HildonFieldnotes, HildonSearchPlace, HildonSearchGeocaches, \ + HildonAboutDialog, HildonDownloadMap, HildonToolsDialog +from advancedcaching.portrait import FremantleRotation +from advancedcaching.utils import HTMLManipulations +from advancedcaching.simplegui import SimpleGui, UpdownRows -import logging logger = logging.getLogger('simplegui') + class HildonGui(HildonToolsDialog, HildonSearchPlace, HildonFieldnotes, HildonSearchGeocaches, HildonAboutDialog, HildonDownloadMap, SimpleGui): MIN_DRAG_REDRAW_DISTANCE = 2 @@ -799,7 +793,7 @@ def _on_show_more_logs(self, widget, logs, events): first_log = 10 + tab_number*logs_in_page #10=number of logs shown in basic view last_log = first_log + logs_in_page if last_log >= len(logs): - last_log = len(logs) + last_log = len(logs) label = str(first_log+1) +"-"+ str(last_log) #add 1 to showing first log, so they are more human readable notebook.append_page(page, gtk.Label(label)) @@ -1159,12 +1153,12 @@ def _generate_list_of_logs(self, logs, events, dialog=False): w_text.set_alignment(0, 0) if events != None: - events.append(self.window.connect('configure-event', self._on_configure_label, w_text, True)) + events.append(self.window.connect('configure-event', self._on_configure_label, w_text, True)) if dialog: #inside dialog (not window) - w_text.set_size_request(self.window.size_request()[0] - 60, -1) + w_text.set_size_request(self.window.size_request()[0] - 60, -1) else: - w_text.set_size_request(self.window.size_request()[0] - 10, -1) + w_text.set_size_request(self.window.size_request()[0] - 10, -1) w_first = gtk.HBox() w_first.pack_start(w_type, False, False) diff --git a/advancedcaching/openstreetmap.py b/advancedcaching/openstreetmap.py index c897b24..d57d1af 100644 --- a/advancedcaching/openstreetmap.py +++ b/advancedcaching/openstreetmap.py @@ -23,22 +23,19 @@ from __future__ import with_statement import logging -logger = logging.getLogger('openstreetmap') - from os import path, mkdir, extsep, remove +from socket import setdefaulttimeout from threading import Semaphore from urllib import urlretrieve -from socket import setdefaulttimeout -import connection -setdefaulttimeout(30) - - -CONCURRENT_THREADS = 20 +from advancedcaching import connection +logger = logging.getLogger('openstreetmap') +setdefaulttimeout(30) - +CONCURRENT_THREADS = 20 + def get_tile_loader(prefix, remote_url, max_zoom = 18, reverse_zoom = False, file_type = 'png', size = 256): class TileLoader(): diff --git a/advancedcaching/provider.py b/advancedcaching/provider.py index 0194c21..33f5025 100644 --- a/advancedcaching/provider.py +++ b/advancedcaching/provider.py @@ -20,11 +20,13 @@ # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching # +import logging + +from copy import copy from math import sqrt from sqlite3 import connect, Row -from copy import copy -import logging + logger = logging.getLogger(__name__) diff --git a/advancedcaching/qml/DefaultTextDialog.qml b/advancedcaching/qml/DefaultTextDialog.qml new file mode 100644 index 0000000..633093f --- /dev/null +++ b/advancedcaching/qml/DefaultTextDialog.qml @@ -0,0 +1,50 @@ +import com.nokia.meego 1.0 +import QtQuick 1.1 + +Sheet { + id: test + //anchors.centerIn: parent + acceptButtonText: "Save" + rejectButtonText: "Close" + //titleText: "Edit Coordinate" + + + content: [ + MouseArea { // to keep the dialog from closing when the user clicks on the background + anchors.fill: cs + onClicked: { } + }, + + Label { + id: intro + text: "This is the default text presented to you when creating a field note or log. You can use the following placeholders: %(machine)s = device name, %c = Date and Time, %x = Date, %X = Time and more, just search the web for strftime." + anchors.top: parent.top + anchors.left: parent.left + anchors.right: parent.right + anchors.leftMargin: 16 + anchors.rightMargin: 16 + anchors.topMargin: 16 + }, + + TextArea { + id: fieldnoteText + anchors.top: intro.bottom + anchors.left: parent.left + anchors.right: parent.right + anchors.leftMargin: 16 + anchors.rightMargin: 16 + anchors.bottom: parent.bottom + anchors.topMargin: 16 + anchors.bottomMargin: 16 + textFormat: TextEdit.PlainText + wrapMode: TextEdit.Wrap + text: settings.optionsDefaultLogText + + } + + ] + + function getValue() { + return fieldnoteText.text; + } +} diff --git a/advancedcaching/qml/DetailsDefaultPage.qml b/advancedcaching/qml/DetailsDefaultPage.qml index 5a9b5e4..af32a07 100644 --- a/advancedcaching/qml/DetailsDefaultPage.qml +++ b/advancedcaching/qml/DetailsDefaultPage.qml @@ -152,7 +152,7 @@ Page { ListButton { - text: "Fieldnote/Share" + text: "Log/Fieldnote/Share" onClicked: { pageFieldnotes.source = "FieldnotesPage.qml"; diff --git a/advancedcaching/qml/FieldnoteDialog.qml b/advancedcaching/qml/FieldnoteDialog.qml new file mode 100644 index 0000000..0381e92 --- /dev/null +++ b/advancedcaching/qml/FieldnoteDialog.qml @@ -0,0 +1,40 @@ +import com.nokia.meego 1.0 +import QtQuick 1.1 + +Sheet { + id: test + acceptButtonText: "Save" + rejectButtonText: "Close" + + + content: [ + MouseArea { // to keep the dialog from closing when the user clicks on the background + anchors.fill: test + onClicked: { } + }, + + + TextArea { + id: fieldnoteText + anchors.top: parent.top + anchors.left: parent.left + anchors.right: parent.right + anchors.leftMargin: 16 + anchors.rightMargin: 16 + anchors.bottom: parent.bottom + anchors.topMargin: 16 + anchors.bottomMargin: 16 + textFormat: TextEdit.PlainText + wrapMode: TextEdit.Wrap + } + + ] + + function getValue() { + return fieldnoteText.text; + } + + function setValue(text) { + fieldnoteText.text = text; + } +} diff --git a/advancedcaching/qml/FieldnotesPage.qml b/advancedcaching/qml/FieldnotesPage.qml index 257e56a..71cff74 100644 --- a/advancedcaching/qml/FieldnotesPage.qml +++ b/advancedcaching/qml/FieldnotesPage.qml @@ -21,17 +21,12 @@ Page { anchors.topMargin: 8 anchors.top: header.bottom - Label { - font.pixelSize: UI.FONT_DEFAULT - text: "Write Fieldnote" - anchors.left: parent.left - wrapMode: Text.Wrap - + + Row { + spacing: 16 Image { + id: fieldnoteHelp source: "image://theme/icon-m-content-description" + (theme.inverted ? "-inverse" : "") - anchors.left: parent.right - anchors.leftMargin: 16 - anchors.verticalCenter: parent.verticalCenter height: 36 width: 36 MouseArea { @@ -39,8 +34,31 @@ Page { onClicked: { infoDialog.open() } } } + + Label { + font.pixelSize: UI.FONT_DEFAULT + text: "Fieldnote" + wrapMode: Text.Wrap + } + + Switch { + onCheckedChanged: { + // GeocacheCoordinate.UPLOAD_AS_FIELDNOTE == 0 + // GeocacheCoordinate.UPLOAD_AS_LOG == 1 + var value = checked ? 1 : 0; + if (currentGeocache.uploadAs != value) currentGeocache.uploadAs = value; + } + checked: (currentGeocache.uploadAs == 1) + + } + + Label { + font.pixelSize: UI.FONT_DEFAULT + text: "Log Entry" + wrapMode: Text.Wrap + } } - + Item { anchors.left: parent.left @@ -86,24 +104,21 @@ Page { anchors.topMargin: 8 anchors.bottom: row1.top anchors.bottomMargin: 8 - onActiveFocusChanged: { - if (! activeFocus) { - saveFieldnote(); - } - } textFormat: TextEdit.PlainText wrapMode: TextEdit.Wrap text: currentGeocache.fieldnotes + enabled: false anchors.leftMargin: 16 anchors.rightMargin: 16 + } Row { id: row1 Button { - text: "Upload all Fieldnotes now" - width: 4 * parent.width/5 + text: "Upload all Logs/Fieldnotes now" + width: parent.width onClicked: { controller.uploadFieldnotes(); } @@ -122,12 +137,9 @@ Page { model: logModel onSelectedIndexChanged: { if (selectedIndex != currentGeocache.logas) { - saveFieldnote(); + currentGeocache.logas = Math.max(logAsDialog.selectedIndex, 0); } } - onAccepted: { - console.debug("HI!"); - } } QueryDialog { @@ -140,8 +152,22 @@ Page { anchors.left: parent.left anchors.right: parent.right text: "Fieldnotes are temporary log entries, which can be reviewed and submitted as regular logs later on.

After uploading, you will find them in your account overview on the web page. If you don't upload them now, they are stored here for later uploading." + color: UI.COLOR_DIALOG_TEXT }] } + + + MouseArea { + anchors.fill: fieldnoteText + onClicked: { + fieldnoteDialogLoader.source = "FieldnoteDialog.qml"; + fieldnoteDialogLoader.item.accepted.connect(function() { + currentGeocache.fieldnotes = fieldnoteDialogLoader.item.getValue(); + }); + fieldnoteDialogLoader.item.setValue(currentGeocache.fieldnotes ? currentGeocache.fieldnotes : settings.getFieldnoteDefaultText()); + fieldnoteDialogLoader.item.open(); + } + } Connections { target: rootWindow @@ -153,22 +179,22 @@ Page { ListModel { id: logModel - ListElement{ name: "Don't upload Fieldnote" } + ListElement{ name: "Don't upload" } ListElement{ name: "Found it!" } ListElement{ name: "Didn't find it!" } ListElement{ name: "Write a note" } } - function saveFieldnote() { - var logas = Math.max(logAsDialog.selectedIndex, 0) - var text = fieldnoteText.text - currentGeocache.setFieldnote(logas, text) - } - function openMenu() { menu.open(); } + + + Loader { + id: fieldnoteDialogLoader + } + Menu { id: menu diff --git a/advancedcaching/qml/ListPage.qml b/advancedcaching/qml/ListPage.qml index 9e2096d..6636731 100644 --- a/advancedcaching/qml/ListPage.qml +++ b/advancedcaching/qml/ListPage.qml @@ -29,11 +29,11 @@ Page { } ListButton { - text: "...having Fieldnotes" + text: "...having Logs/Fieldnotes" onClicked: { pageGeocacheList.source = "GeocacheListPage.qml"; - pageGeocacheList.item.title = "Geocaches with Fieldnotes"; + pageGeocacheList.item.title = "Geocaches with Logs/Notes"; pageGeocacheList.item.model = controller.getGeocachesWithFieldnotes(); pageGeocacheList.item.model.sort(0, gps); showDetailsPage(pageGeocacheList.item); diff --git a/advancedcaching/qml/SettingsPage.qml b/advancedcaching/qml/SettingsPage.qml index a117b2f..d41591a 100644 --- a/advancedcaching/qml/SettingsPage.qml +++ b/advancedcaching/qml/SettingsPage.qml @@ -313,12 +313,32 @@ Page { } } + Label { font.pixelSize: 20 wrapMode: Text.Wrap width: col1.width text: "AGTL skips geocaches which have already been updated in the last " + settings.optionsRedownloadAfter + " day(s). Set to zero to always update all geocaches on the map." } + + Label { + font.pixelSize: 20 + color: UI.COLOR_INFOLABEL + text: "Fieldnotes and Logs" + } + + Button { + anchors.right: parent.right + text: "Change default text" + onClicked: { + // defaultTextDialogLoader + defaultTextDialogLoader.source = "DefaultTextDialog.qml"; + defaultTextDialogLoader.item.accepted.connect(function() { + settings.optionsDefaultLogText = defaultTextDialogLoader.item.getValue(); + }); + defaultTextDialogLoader.item.open() + } + } Label { font.pixelSize: 20 @@ -376,6 +396,9 @@ Page { } + Loader { + id: defaultTextDialogLoader + } ToolBarLayout { id: settingsTools diff --git a/advancedcaching/qmlgui.py b/advancedcaching/qmlgui.py index a42532f..83a42c5 100644 --- a/advancedcaching/qmlgui.py +++ b/advancedcaching/qmlgui.py @@ -21,24 +21,22 @@ # import logging -logger = logging.getLogger('qmlgui') - -from PySide.QtGui import QApplication -from PySide.QtDeclarative import QDeclarativeView -from PySide.QtOpenGL import QGLWidget -from PySide import QtCore import os -import sys -import geo -geo.DEGREES = geo.DEGREES.decode('utf-8') import re -import geocaching -import gpsreader +import sys from os import path -from gui import Gui -#from astral import Astral +from PySide import QtCore +from PySide.QtDeclarative import QDeclarativeView +from PySide.QtGui import QApplication +from PySide.QtOpenGL import QGLWidget + +from advancedcaching import geo, geocaching, gpsreader +from advancedcaching.gui import Gui + + +logger = logging.getLogger('qmlgui') +geo.DEGREES = geo.DEGREES.decode('utf-8') -d = lambda x: x#.decode('utf-8', 'replace') class Controller(QtCore.QObject): @@ -239,15 +237,15 @@ def _progress_message(self): return self._progress_message def _core_version(self): - import core + from advancedcaching import core return core.VERSION def _parser_version(self): - import cachedownloader + from advancedcaching import cachedownloader return cachedownloader.VERSION def _parser_date(self): - import cachedownloader + from advancedcaching import cachedownloader return cachedownloader.VERSION_DATE @QtCore.Slot() @@ -795,6 +793,15 @@ def _map_types(self): def createSetting(name, type, signal, inputNotify = True): return QtCore.Property(type, lambda x: x._setting(name, type), lambda x, m: x._set_setting(name, m, inputNotify), notify=signal) + + @QtCore.Slot(result=str) + def getFieldnoteDefaultText(self): + from time import gmtime + from time import localtime + from time import strftime + + self._last_fieldnote_text = strftime(self.settings['options_default_log_text'], localtime()) % {'machine': 'Nokia N9'} + return self._last_fieldnote_text mapPositionLat = createSetting('map_position_lat', float, settingsChanged, False) mapPositionLon = createSetting('map_position_lon', float, settingsChanged, False) @@ -811,6 +818,7 @@ def createSetting(name, type, signal, inputNotify = True): debugLogToHTTP = createSetting('debug_log_to_http', bool, settingsChanged) optionsRedownloadAfter = createSetting('options_redownload_after', int, settingsChanged) downloadNotFound = createSetting('download_not_found', bool, settingsChanged) + optionsDefaultLogText = createSetting('options_default_log_text', str, settingsChanged) currentMapType = QtCore.Property(QtCore.QObject, _get_current_map_type, _set_current_map_type, notify=settingsChanged) mapTypes = QtCore.Property(QtCore.QObject, _map_types, notify=settingsChanged) @@ -847,10 +855,10 @@ def _lon(self): return self._coordinate.lon if self._is_valid else -1 def _display_text(self): - return d(self._coordinate.display_text) + return self._coordinate.display_text def _comment(self): - return d(self._coordinate.comment) + return self._coordinate.comment def _user_coordinate_id(self): return self._coordinate.user_coordinate_id if self._coordinate.user_coordinate_id != None else -1 @@ -1005,7 +1013,7 @@ def _shortdesc(self): return self._geocache.shortdesc def _stripped_shortdesc(self): - from utils import HTMLManipulations + from advancedcaching.utils import HTMLManipulations return HTMLManipulations.strip_html_visual(self._geocache.shortdesc) def _desc(self): @@ -1086,10 +1094,38 @@ def _logas(self): return int(self._geocache.logas) except ValueError: return 0 + + def _set_logas(self, logas): + from time import gmtime + from time import strftime + self._geocache.logas = logas + self._geocache.logdate = strftime('%Y-%m-%d', gmtime()) + self.core.save_fieldnote(self._geocache) + self.changed.emit() + + + def _upload_as(self): + try: + return int(self._geocache.upload_as) + except (TypeError, ValueError): + return 0 + + def _set_upload_as(self, upload_as): + self._geocache.upload_as = upload_as + self.core.save_fieldnote(self._geocache) + self.changed.emit() def _fieldnotes(self): return self._geocache.fieldnotes + def _set_fieldnotes(self, text): + from time import gmtime + from time import strftime + self._geocache.fieldnotes = text + self._geocache.logdate = strftime('%Y-%m-%d', gmtime()) + self.core.save_fieldnote(self._geocache) + self.changed.emit() + def _marked(self): return self._geocache.marked @@ -1136,16 +1172,7 @@ def _calc_coordinates(self): return self._calc_coordinate_list - @QtCore.Slot(str, str) - def setFieldnote(self, logas, text): - from time import gmtime - from time import strftime - logger.debug("Setting fieldnote, logas=%r, text=%r" % (logas, text)) - self._geocache.logas = logas - self._geocache.fieldnotes = text - self._geocache.logdate = strftime('%Y-%m-%d', gmtime()) - self.core.save_fieldnote(self._geocache) - self.changed.emit() + @QtCore.Slot() def setViewed(self): @@ -1179,11 +1206,12 @@ def setViewed(self): coordinatesCount = QtCore.Property(int, _coordinates_count, notify=coordsChanged) hasDetails = QtCore.Property(bool, _has_details, notify=changed) hints = QtCore.Property(str, _hints, notify=changed) - logas = QtCore.Property(int, _logas, notify=changed) - fieldnotes = QtCore.Property(str, _fieldnotes, notify=changed) + logas = QtCore.Property(int, _logas, _set_logas, notify=changed) + fieldnotes = QtCore.Property(str, _fieldnotes, _set_fieldnotes, notify=changed) varList = QtCore.Property(QtCore.QObject, _var_list, notify=coordsChanged) attributes = QtCore.Property(str, _attributes, notify=changed) calcCoordinates = QtCore.Property(QtCore.QObject, _calc_coordinates, notify=coordsChanged) + uploadAs = QtCore.Property(int, _upload_as, _set_upload_as, notify=changed) class GeocacheListModel(QtCore.QAbstractListModel): COLUMNS = ('geocache',) diff --git a/advancedcaching/simplegui.py b/advancedcaching/simplegui.py index 77b77ad..e403a1a 100644 --- a/advancedcaching/simplegui.py +++ b/advancedcaching/simplegui.py @@ -18,7 +18,6 @@ # Author: Daniel Fett agtl@danielfett.de # Jabber: fett.daniel@jaber.ccc.de # Bugtracker and GIT Repository: http://github.com/webhamster/advancedcaching -# # deps: python-html python-image python-netclient python-misc python-pygtk python-mime python-json @@ -28,32 +27,32 @@ # add "next waypoint" button? # add description to displayed images? # add translation support? - -### For the gui :-) -import math -from astral import Astral -import geo -import geocaching import gobject import gtk - import logging +import math +import pango +import re +from os import extsep +from os.path import islink, realpath, dirname, abspath, join, exists + +from advancedcaching import geo, geocaching +from advancedcaching.astral import Astral +from advancedcaching.constants import TYPE_REGULAR, TYPE_MULTI, TYPE_MYSTERY, TYPE_VIRTUAL +from advancedcaching.gtkmap import Map, GeocacheLayer, MarksLayer, OsdLayer +from advancedcaching.gui import Gui +from advancedcaching.utils import HTMLManipulations + + logger = logging.getLogger('simplegui') + try: import gtk.glade - import extListview + from advancedcaching import extListview except (ImportError): logger.info( "Please install glade if you're NOT on the maemo platform.") -import pango -from os import extsep -from os.path import islink, realpath, dirname, abspath, join, exists -import re -from utils import HTMLManipulations -from gtkmap import Map, GeocacheLayer, MarksLayer, OsdLayer -from gui import Gui - class SimpleGui(Gui): @@ -265,10 +264,10 @@ def load_ui(self): self.search_elements = { 'type': { - geocaching.GeocacheCoordinate.TYPE_REGULAR: xml.get_widget('check_search_type_traditional'), - geocaching.GeocacheCoordinate.TYPE_MULTI: xml.get_widget('check_search_type_multi'), - geocaching.GeocacheCoordinate.TYPE_MYSTERY: xml.get_widget('check_search_type_unknown'), - geocaching.GeocacheCoordinate.TYPE_VIRTUAL: xml.get_widget('check_search_type_virtual'), + TYPE_REGULAR: xml.get_widget('check_search_type_traditional'), + TYPE_MULTI: xml.get_widget('check_search_type_multi'), + TYPE_MYSTERY: xml.get_widget('check_search_type_unknown'), + TYPE_VIRTUAL: xml.get_widget('check_search_type_virtual'), 'all': xml.get_widget('check_search_type_other') }, 'name': xml.get_widget('entry_search_name'), @@ -800,10 +799,8 @@ def get_val_from_text(input, use_max): else: return default - types = [a for a in [geocaching.GeocacheCoordinate.TYPE_REGULAR, - geocaching.GeocacheCoordinate.TYPE_MULTI, - geocaching.GeocacheCoordinate.TYPE_MYSTERY, - geocaching.GeocacheCoordinate.TYPE_VIRTUAL] if self.search_elements['type'][a].get_active()] + types = [a for a in [TYPE_REGULAR, TYPE_MULTI, TYPE_MYSTERY, TYPE_VIRTUAL] + if self.search_elements['type'][a].get_active()] if self.search_elements['type']['all'].get_active() or len(types) == 0: types = None diff --git a/advancedcaching/utils.py b/advancedcaching/utils.py index 0ef4db6..b7e6f06 100644 --- a/advancedcaching/utils.py +++ b/advancedcaching/utils.py @@ -96,7 +96,7 @@ def _rot13(text): import logging logger = logging.getLogger('utils') - import colorer + from advancedcaching import colorer logger.setLevel(logging.DEBUG) logging.basicConfig(level=logging.DEBUG, format='%(relativeCreated)6d %(levelname)10s %(name)-20s %(message)s',