Add OpenStreetMaps to reverse geo lookup

Location data can be checked from OpenStreetMaps too

README.md

@@ -27,7 +27,7 @@ reverse_geolocate.py [-h] -x
     [XMP SOURCE FOLDER [XMP SOURCE FOLDER ...]]
     [-l LIGHTROOM FOLDER] [-s]
     [-f <overwrite, location, city, state, country, countrycode>]
-    [-g GOOGLE API KEY] [-n] [-v] [--debug] [--test]
+    [-g GOOGLE API KEY] [-o] [-n] [-v] [--debug] [--test]
 
 ### Arguments
 
@@ -38,6 +38,7 @@ Argument | Argument Value | Description
 -s, --strict | | Do strict check for Lightroom files and include the path into the check
 -f, --field | Keyword: overwrite, location, city, state, country, countrycode | In the default no data is overwritten if it is already set. With the 'overwrite' flag all data is set new from the Google Maps location data. Other arguments are each of the location fields and if set only this field will be set. This can be combined with the 'overwrite' flag to overwrite already set data
 -n, --nobackup | | Do not create a backup of XMP sidecar file when it is changed
+-o, --openstreetmap | | Use OpenStreetMap instead of the default google maps
 -g, --google | Google Maps API Key | If available, to avoid the access limitations to the reverse location lookup
 -v, --verbose | | More verbose output. Currently not used
 --debug | | Full detailed debug output. Will print out alot of data
@@ -75,6 +76,23 @@ order | type | target set
 6 | sublocality_level_2 | Location
 7 | route | Location
 
+### OpenStreetMap data priority
+
+order | type | target set
+--- | --- | ---
+1 | country_code | CountryCode
+2 | country | Country
+3 | state | State
+4 | city | City
+5 | city_district | City
+6 | state_district | City
+7 | county | Location
+8 | town | Location
+9 | suburb | Location
+10 | hamlet | Location
+11 | neighbourhood | Location
+12 | raod | Location
+
 ### Script stats and errors on update
 
 After the script is done the following overview will be printed

bin/reverse_geolocate.py

@@ -62,32 +62,35 @@ class readable_dir(argparse.Action):
 ### MAIN FUNCTIONS
 
 # METHOD: reverseGeolocate
-# PARAMS: latitude, longitude
+# PARAMS: latitude, longitude, map search target (google or openstreetmap)
-# RETURN: dict with location, city, state, country, country code
+# RETURN: dict with all data (see below)
-#         if not fillable, entry is empty
+# DESC  : wrapper to call to either the google or openstreetmap
-#         google images lookup base settings
+def reverseGeolocate(longitude, latitude, map_type):
-# SAMPLE: http://maps.googleapis.com/maps/api/geocode/json?latlng=35.6671355,139.7419185&sensor=false
-def reverseGeolocate(longitude, latitude):
     # clean up long/lat
     # they are stored with N/S/E/W if they come from an XMP
     # format: Deg,Min.Sec[NSEW]
     # NOTE: lat is N/S, long is E/W
     # detect and convert
     lat_long = longLatReg(longitude = longitude, latitude = latitude)
-    # sensor (why?)
+    # which service to use
-    sensor = 'false'
+    if map_type == 'google':
-    # request to google
+        return reverseGeolocateGoogle(lat_long['longitude'], lat_long['latitude'])
-    # if a google api key is used, the request has to be via https
+    elif map_type == 'openstreetmap':
-    protocol = 'https://' if args.google_api_key else 'http://'
+        return reverseGeolocateOpenStreetMap(lat_long['longitude'], lat_long['latitude'])
-    base = "maps.googleapis.com/maps/api/geocode/json?"
+    else:
-    # build the base params
+        return {
-    params = "latlng={lat},{lon}&sensor={sensor}".format(lon = lat_long['longitude'], lat = lat_long['latitude'], sensor = sensor)
+            'Country': '',
-    # if we have a google api key, add it here
+            'status': 'ERROR',
-    key = "&key={}".format(args.google_api_key) if args.google_api_key else ''
+            'error': 'Map type not valid'
-    # build the full url and send it to google
+        }
-    url = "{protocol}{base}{params}{key}".format(protocol = protocol, base = base, params = params, key = key)
+
-    response = requests.get(url)
+# METHOD: reverseGeolocateInit
-    # loop through the json response to get the best matching entry
+# PARAMS: longitude, latitude
+# RETURN: empty geolocation dictionary, or error flag if lat/long is not valid
+# DESC  : inits the dictionary for return, and checks the lat/long on valid
+#         returns geolocation dict with status = 'ERROR' if an error occurded
+def reverseGeolocateInit(longitude, latitude):
+    # basic dict format
     geolocation = {
         'CountryCode': '',
         'Country': '',
@@ -98,10 +101,92 @@ def reverseGeolocate(longitude, latitude):
         'status': '',
         'error_message': ''
     }
+    # error if long/lat is not valid
+    latlong_re = re.compile('^\d+\.\d+$')
+    if not latlong_re.match(str(longitude)) or not latlong_re.match(str(latitude)):
+        geolocation['status'] = 'ERROR'
+        geolocation['error_message'] = 'Latitude {} or Longitude {} are not valid'.format(latitude, longitude)
+    return geolocation
+
+# METHOD: reverseGeolocateOpenStreetMap
+# PARAMS: latitude, longitude
+# RETURN: OpenStreetMap reverse lookcation lookup
+#         dict with locaiton, city, state, country, country code
+#         if not fillable, entry is empty
+# SAMPLE: https://nominatim.openstreetmap.org/reverse.php?format=jsonv2&lat=<latitude>&lon=<longitude>&zoom=21&accept-languge=en-US,en&
+def reverseGeolocateOpenStreetMap(longitude, latitude):
+    # init
+    geolocation = reverseGeolocateInit(longitude, latitude)
+    if geolocation['status'] == 'ERROR':
+        return geolocation
+    # query format
+    query_format = 'jsonv2'
+    # language to return (english)
+    language = 'en-US,en'
+    # build query
+    base = 'https://nominatim.openstreetmap.org/reverse.php?'
+    params = 'format={format}&lat={lat}&lon={lon}&accept-language={lang}&zoom=21'.format(lon = longitude, lat = latitude, format = query_format, lang = language)
+    url = "{base}{params}".format(base = base, params = params)
+    response = requests.get(url)
+    # debug output
     if args.debug:
-        print("Search for Lat: {}, Long: {}".format(lat_long['latitude'], lat_long['longitude']))
+        print("OpenStreetMap search for Lat: {}, Long: {}".format(latitude, longitude))
     if args.debug and args.verbose >= 1:
-        print("Google response: {} => TEXT: {} JSON: {}".format(response, response.text, response.json()))
+        print("OpenStreetMap response: {} => JSON: {}".format(response, response.json()))
+    # type map
+    # Country to Location and for each in order of priority
+    type_map = {
+        'CountryCode': ['country_code'],
+        'Country': ['country'],
+        'State': ['state'],
+        'City': ['city', 'city_district', 'state_district'],
+        'Location': ['county', 'town', 'suburb', 'hamlet', 'neighbourhood', 'road']
+    }
+    # if not error
+    if 'error' not in response.json():
+        # get address block
+        addr = response.json()['address']
+        # loop for locations
+        for loc_index in type_map:
+            for index in type_map[loc_index]:
+                if index in addr and not geolocation[loc_index]:
+                    geolocation[loc_index] = addr[index]
+    else:
+        geolocation['status'] = 'ERROR'
+        geolocation['error_message'] = response.json()['error']
+        print("Error in request: {}".format(geolocation['error']))
+    # return
+    return geolocation
+
+# METHOD: reverseGeolocateGoogle
+# PARAMS: latitude, longitude
+# RETURN: Google Maps reverse location lookup
+#         dict with location, city, state, country, country code
+#         if not fillable, entry is empty
+# SAMPLE: http://maps.googleapis.com/maps/api/geocode/json?latlng=<latitude>,<longitude>&sensor=false&key=<api key>
+def reverseGeolocateGoogle(longitude, latitude):
+    # init
+    geolocation = reverseGeolocateInit(longitude, latitude)
+    if geolocation['status'] == 'ERROR':
+        return geolocation
+    # sensor (why?)
+    sensor = 'false'
+    # request to google
+    # if a google api key is used, the request has to be via https
+    protocol = 'https://' if args.google_api_key else 'http://'
+    base = "maps.googleapis.com/maps/api/geocode/json?"
+    # build the base params
+    params = "latlng={lat},{lon}&sensor={sensor}".format(lon = longitude, lat = latitude, sensor = sensor)
+    # if we have a google api key, add it here
+    key = "&key={}".format(args.google_api_key) if args.google_api_key else ''
+    # build the full url and send it to google
+    url = "{protocol}{base}{params}{key}".format(protocol = protocol, base = base, params = params, key = key)
+    response = requests.get(url)
+    # debug output
+    if args.debug:
+        print("Google search for Lat: {}, Long: {}".format(longitude, latitude))
+    if args.debug and args.verbose >= 1:
+        print("Google response: {} => JSON: {}".format(response, response.json()))
     # print("Error: {}".format(response.json()['status']))
     if response.json()['status'] == 'OK':
         # first entry for type = premise
@@ -115,34 +200,37 @@ def reverseGeolocate(longitude, latitude):
                     # -> administrative_area (1, 2),
                     # -> locality,
                     # -> sublocality (_level_1 or 2 first found, then route)
-                    for addr in entry['address_components']:
+                    # so we get the data in the correct order
-                        # print("Addr: {}".format(addr))
+                    for index in ['country', 'administrative_area_level_1', 'administrative_area_level_2', 'locality', 'sublocality_level_1', 'sublocality_level_2', 'route']:
-                        # country code + country
+                        # loop through the entries in the returned json and find matching
-                        if 'country' in addr['types'] and not geolocation['CountryCode']:
+                        for addr in entry['address_components']:
-                            geolocation['CountryCode'] = addr['short_name']
+                            # print("Addr: {}".format(addr))
-                            geolocation['Country'] = addr['long_name']
+                            # country code + country
-                        # state
+                            if index == 'country' and index in addr['types'] and not geolocation['CountryCode']:
-                        if 'administrative_area_level_1' in addr['types'] and not geolocation['State']:
+                                geolocation['CountryCode'] = addr['short_name']
-                            geolocation['State'] = addr['long_name']
+                                geolocation['Country'] = addr['long_name']
-                        if 'administrative_area_level_2' in addr['types'] and not geolocation['State']:
+                            # state
-                            geolocation['State'] = addr['long_name']
+                            if index == 'administrative_area_level_1' and index  in addr['types'] and not geolocation['State']:
-                        # city
+                                geolocation['State'] = addr['long_name']
-                        if 'locality' in addr['types'] and not geolocation['City']:
+                            if index == 'administrative_area_level_2' and index  in addr['types'] and not geolocation['State']:
-                            geolocation['City'] = addr['long_name']
+                                geolocation['State'] = addr['long_name']
-                        # location
+                            # city
-                        if 'sublocality_level_1' in addr['types'] and not geolocation['Location']:
+                            if index == 'locality' and index  in addr['types'] and not geolocation['City']:
-                            geolocation['Location'] = addr['long_name']
+                                geolocation['City'] = addr['long_name']
-                        if 'sublocality_level_2' in addr['types'] and not geolocation['Location']:
+                            # location
-                            geolocation['Location'] = addr['long_name']
+                            if index == 'sublocality_level_1' and index  in addr['types'] and not geolocation['Location']:
-                        # if all failes try route
+                                geolocation['Location'] = addr['long_name']
-                        if 'route' in addr['types'] and not geolocation['Location']:
+                            if index == 'sublocality_level_2' and index  in addr['types'] and not geolocation['Location']:
-                            geolocation['Location'] = addr['long_name']
+                                geolocation['Location'] = addr['long_name']
+                            # if all failes try route
+                            if index == 'route' and index  in addr['types'] and not geolocation['Location']:
+                                geolocation['Location'] = addr['long_name']
         # write OK status
         geolocation['status'] = response.json()['status']
     else:
         geolocation['error_message'] = response.json()['error_message']
         geolocation['status'] = response.json()['status']
-        print("Error in request: {} {}".format(geolocation['status'] , geolocation['error_message']))
+        print("Error in request: {} {}".format(geolocation['status'], geolocation['error_message']))
 
     # return
     return geolocation
@@ -284,6 +372,13 @@ parser.add_argument('-g', '--google',
     help = 'Set a Google API Maps key to overcome the default lookup limitations'
 )
 
+# use open street maps
+parser.add_argument('-o', '--openstreetmap',
+    dest = 'use_openstreetmap',
+    action = 'store_true',
+    help = 'Use openstreetmap instead of Google'
+)
+
 # Do not create backup files
 parser.add_argument('-n', '--nobackup',
     dest = 'no_xmp_backup',
@@ -314,7 +409,14 @@ if not args.verbose:
     args.verbose = 0
 
 if args.debug:
-    print("### ARGUMENT VARS: X: {}, L: {}, F: {}, G: {}, V: {}, D: {}, T: {}".format(args.xmp_sources, args.lightroom_folder, args.field_controls, args.google_api_key, args.verbose, args.debug, args.test))
+    print("### ARGUMENT VARS: X: {}, L: {}, F: {}, M: {}, G: {}, N; {}, V: {}, D: {}, T: {}".format(args.xmp_sources, args.lightroom_folder, args.field_controls, args.use_openstreetmap, args.google_api_key, args.no_xmp_backup, args.verbose, args.debug, args.test))
+
+# set search map type
+map_type = 'google' if not args.use_openstreetmap else 'openstreetmap'
+# if -g and -o, error
+if args.google_api_key and args.use_openstreetmap:
+    print("You cannot set a Google API key and use openstreetmap at the seame time")
+    sys.exit(1)
 
 # The XMP fields const lookup values
 # XML/XMP
@@ -366,7 +468,7 @@ cur = ''
 # count variables
 count = {
     'all': 0,
-    'google': 0,
+    'map': 0,
     'cache': 0,
     'lightroom': 0,
     'changed': 0,
@@ -490,7 +592,7 @@ for xmp_file in work_files:
     # is LR GPS and no XMP GPS => use LR and set XMP
     # same for location names
     # if missing in XMP but in LR -> set in XMP
-    # if missing in both do lookup in Google
+    # if missing in both do lookup in Maps
     if use_lightroom and lightroom_data_ok:
         # check lat/long separate
         if lrdb_row['gpsLatitude'] and not data_set['GPSLatitude']:
@@ -505,7 +607,7 @@ for xmp_file in work_files:
                 data_set[loc] = lrdb_row[loc]
                 if args.debug:
                     print("### -> LR: {} => {}".format(loc, lrdb_row[loc]))
-    # base set done, now check if there is anything unset in the data_set, if yes do a lookup in google
+    # base set done, now check if there is anything unset in the data_set, if yes do a lookup in maps
     # run this through the overwrite checker to get unset if we have a forced overwrite
     has_unset = False
     failed = False
@@ -518,27 +620,27 @@ for xmp_file in work_files:
         if args.debug:
             print("### *** CACHE: {}: {}".format(cache_key, 'NO' if cache_key not in data_cache else 'YES'))
         if cache_key not in data_cache:
-            # get location from google
+            # get location from maps (google or openstreetmap)
-            google_location = reverseGeolocate(latitude = data_set['GPSLatitude'], longitude = data_set['GPSLongitude'])
+            maps_location = reverseGeolocate(latitude = data_set['GPSLatitude'], longitude = data_set['GPSLongitude'], map_type = map_type)
             # cache data with Lat/Long
-            data_cache[cache_key] = google_location
+            data_cache[cache_key] = maps_location
         else:
             # load location from cache
-            google_location = data_cache[cache_key]
+            maps_location = data_cache[cache_key]
             count['cache'] += 1
         # overwrite sets (note options check here)
         if args.debug:
-            print("### Google Location: {}".format(google_location))
+            print("### Map Location ({}): {}".format(map_type, maps_location))
         # must have at least the country set to write anything back
-        if google_location['Country']:
+        if maps_location['Country']:
             for loc in data_set_loc:
                 # only write to XMP if overwrite check passes
                 if checkOverwrite(data_set[loc], loc, args.field_controls):
-                    data_set[loc] = google_location[loc]
+                    data_set[loc] = maps_location[loc]
-                    xmp.set_property(xmp_fields[loc], loc, google_location[loc])
+                    xmp.set_property(xmp_fields[loc], loc, maps_location[loc])
                     write_file = True
             if write_file:
-                count['google'] += 1
+                count['map'] += 1
         else:
             print("(!) Could not geo loaction data ", end = '')
             failed = True
@@ -585,7 +687,7 @@ print("{}".format('=' * 37))
 print("XMP Files found             : {:7,}".format(count['all']))
 print("Updated                     : {:7,}".format(count['changed']))
 print("Skipped                     : {:7,}".format(count['skipped']))
-print("New GeoLocation Google      : {:7,}".format(count['google']))
+print("New GeoLocation from Map    : {:7,}".format(count['map']))
 print("GeoLocation from Cache      : {:7,}".format(count['cache']))
 print("Failed reverse GeoLocate    : {:7,}".format(count['failed']))
 if use_lightroom: