ReverseGeoLocate/reverse-geolocate
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

Reverse Geolocate Clean up run

Browse Source
This commit is contained in:
Clemens Schwaighofer
2022-11-11 23:08:41 +09:00
parent 65e352e8a4
commit b7de6d8119
4 changed files with 785 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{
"python.analysis.typeCheckingMode": "basic"
}
README.md
+16 -4
View File
@@ -4,16 +4,28 @@ Reverse GeoLocate from XMP sidecar files with optional LightRoom DB read
This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database. This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database.
#### Installing and setting up ## Development Setup
```sh
python3 -m venv .venv
source .venv/bin/activate
.venv/bin/python -m pip install -U pip setuptools wheel
```
Then install the requests and python-xmp-toolkit modules from below
## Installing and setting up
The script uses the following external non defauly python libraries The script uses the following external non defauly python libraries
* xmp toolkit * xmp toolkit
* requests * requests
install both with the pip3 command install both with the pip3 command
```
pip3 install requests ```sh
pip3 install python-xmp-toolkit pip install requests
pip install python-xmp-toolkit
``` ```
XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly. XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly.
bin/reverse_geolocate.py
+122 -70
View File
@@ -3,15 +3,14 @@
# AUTHOR : Clemens Schwaighofer # AUTHOR : Clemens Schwaighofer
# DATE : 2018/2/20 # DATE : 2018/2/20
# LICENSE: GPLv3 # LICENSE: GPLv3
# DESC : Set the reverse Geo location (name) from Lat/Long data in XMP files in a lightroom catalogue # DESC :
# Set the reverse Geo location (name) from Lat/Long data in XMP files
# in a lightroom catalogue
# * tries to get pre-set geo location from LR catalog # * tries to get pre-set geo location from LR catalog
# * if not found tries to get data from Google # * if not found tries to get data from Google
# * all data is translated into English with long vowl system (aka ou or oo is ō) # * all data is translated into English with long vowl system (aka ou or oo is ō)
# MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/) # MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/)
import argparse
import sqlite3
import requests
import configparser import configparser
import unicodedata import unicodedata
# import textwrap # import textwrap
@@ -19,10 +18,13 @@ import glob
import os import os
import sys import sys
import re import re
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta import argparse
from libxmp import XMPMeta, consts import sqlite3
from shutil import copyfile, get_terminal_size from shutil import copyfile, get_terminal_size
from math import ceil, radians, sin, cos, atan2, sqrt from math import ceil, radians, sin, cos, atan2, sqrt
import requests
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta
from libxmp import XMPMeta, consts
############################################################## ##############################################################
# FUNCTIONS # FUNCTIONS
@@ -32,13 +34,16 @@ from math import ceil, radians, sin, cos, atan2, sqrt
# this is used by isLatin and onlyLatinChars # this is used by isLatin and onlyLatinChars
cache_latin_letters = {} cache_latin_letters = {}
# ARGPARSE HELPERS # ARGPARSE HELPERS
# call: writable_dir_folder class WritableDirFolder(argparse.Action):
# checks if this is a writeable folder OR file """
# AND it works on nargs * checks if this is a writeable folder OR file
class writable_dir_folder(argparse.Action): AND it works on nargs *
Args:
argparse (_type_): _description_
"""
def __call__(self, parser, namespace, values, option_string=None): def __call__(self, parser, namespace, values, option_string=None):
# we loop through list (this is because of nargs *) # we loop through list (this is because of nargs *)
for prospective_dir in values: for prospective_dir in values:
@@ -632,9 +637,16 @@ def getBackupFileCounter(xmp_file):
return bk_file_counter return bk_file_counter
############################################################## ##############################################################
# ARGUMENT PARSNING # ARGUMENT PARSING
############################################################## ##############################################################
def argument_parser():
"""
Parses the command line arguments
Returns:
Namespace: parsed arguments
"""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files', description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files',
@@ -649,7 +661,7 @@ parser.add_argument(
'--include-source', '--include-source',
required=True, required=True,
nargs='*', nargs='*',
action=writable_dir_folder, action=WritableDirFolder,
dest='xmp_sources', dest='xmp_sources',
metavar='XMP SOURCE FOLDER', metavar='XMP SOURCE FOLDER',
help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here' help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here'
@@ -659,7 +671,7 @@ parser.add_argument(
'-x', '-x',
'--exclude-source', '--exclude-source',
nargs='*', nargs='*',
action=writable_dir_folder, action=WritableDirFolder,
dest='exclude_sources', dest='exclude_sources',
metavar='EXCLUDE XMP SOURCE FOLDER', metavar='EXCLUDE XMP SOURCE FOLDER',
help='Folders and files that will be excluded.' help='Folders and files that will be excluded.'
@@ -688,7 +700,8 @@ parser.add_argument(
# set behaviour override # set behaviour override
# FLAG: default: only set not filled # FLAG: default: only set not filled
# other: overwrite all or overwrite if one is missing, overwrite specifc field (as defined below) # other: overwrite all or overwrite if one is missing,
# overwrite specifc field (as defined below)
# fields: Location, City, State, Country, CountryCode # fields: Location, City, State, Country, CountryCode
parser.add_argument( parser.add_argument(
'-f', '-f',
@@ -698,10 +711,14 @@ parser.add_argument(
choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'], choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'],
dest='field_controls', dest='field_controls',
metavar='<overwrite, location, city, state, country, countrycode>', metavar='<overwrite, location, city, state, country, countrycode>',
help='On default only set fields that are not set yet. Options are: '\ help=(
'Overwrite (write all new), Location, City, State, Country, CountryCode. '\ 'On default only set fields that are not set yet. Options are: '
'Multiple can be given for combination overwrite certain fields only or set only certain fields. '\ 'Overwrite (write all new), Location, City, State, Country, CountryCode. '
'If with overwrite the field will be overwritten if already set, else it will be always skipped.' 'Multiple can be given for combination overwrite certain fields only '
'or set only certain fields. '
'If with overwrite the field will be overwritten if already set, '
'else it will be always skipped.'
)
) )
parser.add_argument( parser.add_argument(
@@ -713,10 +730,12 @@ parser.add_argument(
const='10m', # default is 10m const='10m', # default is 10m
dest='fuzzy_distance', dest='fuzzy_distance',
metavar='FUZZY DISTANCE', metavar='FUZZY DISTANCE',
help='Allow fuzzy distance cache lookup. Optional distance can be given, '\ help=(
'if not set default of 10m is used. '\ 'Allow fuzzy distance cache lookup. Optional distance can be given, '
'if not set default of 10m is used. '
'Allowed argument is in the format of 12m or 12km' 'Allowed argument is in the format of 12m or 12km'
) )
)
# Google Maps API key to overcome restrictions # Google Maps API key to overcome restrictions
parser.add_argument( parser.add_argument(
@@ -760,7 +779,10 @@ parser.add_argument(
'--read-only', '--read-only',
dest='read_only', dest='read_only',
action='store_true', action='store_true',
help='Read current values from the XMP file only, do not read from LR or lookup any data and write back' help=(
'Read current values from the XMP file only, '
'do not read from LR or lookup any data and write back'
)
) )
# only list unset ones # only list unset ones
@@ -818,17 +840,27 @@ parser.add_argument(
) )
# debug flag # debug flag
parser.add_argument('--debug', action='store_true', dest='debug', help='Set detailed debug output') parser.add_argument(
'--debug', action='store_true', dest='debug', help='Set detailed debug output'
)
# test flag # test flag
parser.add_argument('--test', action='store_true', dest='test', help='Do not write data back to file') parser.add_argument(
'--test', action='store_true', dest='test', help='Do not write data back to file'
)
# read in the argumens # read in the argumens
args = parser.parse_args() return parser.parse_args()
############################################################## ##############################################################
# MAIN CODE # MAIN CODE
############################################################## ##############################################################
def main():
"""
Main code run
"""
args = argument_parser()
# init verbose to 0 if not set # init verbose to 0 if not set
if not args.verbose: if not args.verbose:
args.verbose = 0 args.verbose = 0
@@ -840,25 +872,14 @@ if not args.unset_only:
args.unset_only = 0 args.unset_only = 0
if args.debug: if args.debug:
print("### ARGUMENT VARS: I: {incl}, X: {excl}, L: {lr}, F: {fc}, D: {fdist}, M: {osm}, G: {gp}, E: {em}, R: {read}, U: {us}, A: {adj}, C: {cmp}, N: {nbk}, W: {wrc}, V: {v}, D: {d}, T: {t}".format( print(
incl=args.xmp_sources, "### ARGUMENT VARS: "
excl=args.exclude_sources, f"I: {args.xmp_sources}, X: {args.exclude_sources}, L: {args.lightroom_folder}, "
lr=args.lightroom_folder, f"F: {args.field_controls}, D: {args.fuzzy_distance}, M: {args.use_openstreetmap}, "
fc=args.field_controls, f"G: {args.google_api_key}, E: {args.email}, R: {args.read_only}, U: {args.unset_only}, "
fdist=args.fuzzy_distance, f"A: {args.no_autoadjust}, C: {args.compact_view}, N: {args.no_xmp_backup}, "
osm=args.use_openstreetmap, f"W: {args.config_write}, V: {args.verbose}, D: {args.debug}, T: {args.test}"
gp=args.google_api_key, )
em=args.email,
read=args.read_only,
us=args.unset_only,
adj=args.no_autoadjust,
cmp=args.compact_view,
nbk=args.no_xmp_backup,
wrc=args.config_write,
v=args.verbose,
d=args.debug,
t=args.test
))
# error flag # error flag
error = False error = False
@@ -1230,8 +1251,9 @@ for xmp_file in work_files: # noqa: C901
if lrdb_row[loc] and not data_set[loc]: if lrdb_row[loc] and not data_set[loc]:
data_set[loc] = lrdb_row[loc] data_set[loc] = lrdb_row[loc]
if args.debug: if args.debug:
print("### -> LR: {} => {}".format(loc, lrdb_row[loc])) print(f"### -> LR: {loc} => {lrdb_row[loc]}")
# base set done, now check if there is anything unset in the data_set, if yes do a lookup in maps # base set done, now check if there is anything unset in the data_set,
# if yes do a lookup in maps
# run this through the overwrite checker to get unset if we have a forced overwrite # run this through the overwrite checker to get unset if we have a forced overwrite
has_unset = False has_unset = False
failed = False failed = False
@@ -1241,13 +1263,17 @@ for xmp_file in work_files: # noqa: C901
has_unset = True has_unset = True
if has_unset: if has_unset:
# check if lat/long is in cache # check if lat/long is in cache
cache_key = '{}#{}'.format(data_set['GPSLongitude'], data_set['GPSLatitude']) cache_key = f"{data_set['GPSLongitude']}#{data_set['GPSLatitude']}"
if args.debug: if args.debug:
print("### *** CACHE: {}: {}".format(cache_key, 'NO' if cache_key not in data_cache else 'YES')) print(
f"### *** CACHE: {cache_key}: "
f"{'NO' if cache_key not in data_cache else 'YES'}"
)
# main chache check = identical # main chache check = identical
# second cache level check is on distance: # second cache level check is on distance:
# default distance is 10m, can be set via flag # default distance is 10m, can be set via flag
# check distance to previous cache entries (reverse newest to oldest) and match before we do google lookup # check distance to previous cache entries (reverse newest to oldest)
# and match before we do google lookup
if cache_key not in data_cache: if cache_key not in data_cache:
has_fuzzy_cache = False has_fuzzy_cache = False
if args.fuzzy_distance: if args.fuzzy_distance:
@@ -1258,20 +1284,39 @@ for xmp_file in work_files: # noqa: C901
# split up cache key so we can use in the distance calc method # split up cache key so we can use in the distance calc method
to_lat_long = _cache_key.split('#') to_lat_long = _cache_key.split('#')
# get the distance based on current set + cached set # get the distance based on current set + cached set
# print("Lookup f-long {} f-lat {} t-long {} t-lat {}".format(data_set['GPSLongitude'], data_set['GPSLatitude'], to_lat_long[0], to_lat_long[1])) # print(
distance = getDistance(from_longitude=data_set['GPSLongitude'], from_latitude=data_set['GPSLatitude'], to_longitude=to_lat_long[0], to_latitude=to_lat_long[1]) # f"Lookup f-long {data_set['GPSLongitude']} "
# f"f-lat {data_set['GPSLatitude']} "
# f"t-long {to_lat_long[0]} t-lat {to_lat_long[1]}"
# )
distance = getDistance(
from_longitude=data_set['GPSLongitude'],
from_latitude=data_set['GPSLatitude'],
to_longitude=to_lat_long[0],
to_latitude=to_lat_long[1]
)
if args.debug: if args.debug:
print("### **= FUZZY CACHE: => distance: {} (m), shortest: {}".format(distance, shortest_distance)) print(
f"### **= FUZZY CACHE: => distance: {distance} (m), "
f"shortest: {shortest_distance}"
)
if distance <= shortest_distance: if distance <= shortest_distance:
# set new distance and keep current best matching location # set new distance and keep current best matching location
shortest_distance = distance shortest_distance = distance
best_match_latlong = _cache_key best_match_latlong = _cache_key
has_fuzzy_cache = True has_fuzzy_cache = True
if args.debug: if args.debug:
print("### ***= FUZZY CACHE: YES => Best match: {}".format(best_match_latlong)) print(
"### ***= FUZZY CACHE: YES => "
f"Best match: {best_match_latlong}"
)
if not has_fuzzy_cache: if not has_fuzzy_cache:
# get location from maps (google or openstreetmap) # get location from maps (google or openstreetmap)
maps_location = reverseGeolocate(latitude=data_set['GPSLatitude'], longitude=data_set['GPSLongitude'], map_type=map_type) maps_location = reverseGeolocate(
latitude=data_set['GPSLatitude'],
longitude=data_set['GPSLongitude'],
map_type=map_type
)
# cache data with Lat/Long # cache data with Lat/Long
data_cache[cache_key] = maps_location data_cache[cache_key] = maps_location
from_cache = False from_cache = False
@@ -1290,7 +1335,7 @@ for xmp_file in work_files: # noqa: C901
from_cache = True from_cache = True
# overwrite sets (note options check here) # overwrite sets (note options check here)
if args.debug: if args.debug:
print("### Map Location ({}): {}".format(map_type, maps_location)) print(f"### Map Location ({map_type}): {maps_location}")
# must have at least the country set to write anything back # must have at least the country set to write anything back
if maps_location['Country']: if maps_location['Country']:
for loc in data_set_loc: for loc in data_set_loc:
@@ -1306,7 +1351,7 @@ for xmp_file in work_files: # noqa: C901
failed = True failed = True
else: else:
if args.debug: if args.debug:
print("Lightroom data use: {}, Lightroom data ok: {}".format(use_lightroom, lightroom_data_ok)) print(f"Lightroom data use: {use_lightroom}, Lightroom data ok: {lightroom_data_ok}")
# check if the data_set differs from the original (LR db load) # check if the data_set differs from the original (LR db load)
# if yes write, else skip # if yes write, else skip
if use_lightroom and lightroom_data_ok: if use_lightroom and lightroom_data_ok:
@@ -1350,25 +1395,32 @@ if use_lightroom:
lrdb.close() lrdb.close()
# end stats only if we write # end stats only if we write
print("{}".format('=' * 40)) print(f"{'=' * 40}")
print("XMP Files found : {:9,}".format(count['all'])) print(f"XMP Files found : {count['all']:9,}")
if args.read_only: if args.read_only:
print("XMP Files listed : {:9,}".format(count['listed'])) print(f"XMP Files listed : {count['listed']:9,}")
if not args.read_only: if not args.read_only:
print("Updated : {:9,}".format(count['changed'])) print(f"Updated : {count['changed']:9,}")
print("Skipped : {:9,}".format(count['skipped'])) print(f"Skipped : {count['skipped']:9,}")
print("New GeoLocation from Map : {:9,}".format(count['map'])) print(f"New GeoLocation from Map : {count['map']:9,}")
print("GeoLocation from Cache : {:9,}".format(count['cache'])) print(f"GeoLocation from Cache : {count['cache']:9,}")
print("GeoLocation from Fuzzy Cache : {:9,}".format(count['fuzzy_cache'])) print(f"GeoLocation from Fuzzy Cache : {count['fuzzy_cache']:9,}")
print("Failed reverse GeoLocate : {:9,}".format(count['failed'])) print(f"Failed reverse GeoLocate : {count['failed']:9,}")
if use_lightroom: if use_lightroom:
print("GeoLocaction from Lightroom : {:9,}".format(count['lightroom'])) print(f"GeoLocaction from Lightroom : {count['lightroom']:9,}")
print("No Lightroom data found : {:9,}".format(count['not_found'])) print(f"No Lightroom data found : {count['not_found']:9,}")
print("More than one found in LR : {:9,}".format(count['many_found'])) print(f"More than one found in LR : {count['many_found']:9,}")
# if we have failed data # if we have failed data
if len(failed_files) > 0: if len(failed_files) > 0:
print("{}".format('-' * 40)) print(f"{'-' * 40}")
print("Files that failed to update:") print("Files that failed to update:")
print("{}".format(', '.join(failed_files))) print(f"{', '.join(failed_files)}")
##############################################################
# MAIN RUN
##############################################################
main()
# __END__ # __END__
requirement.txt
+8
View File
@@ -0,0 +1,8 @@
certifi==2022.9.24
charset-normalizer==2.1.1
idna==3.4
install==1.3.5
python-xmp-toolkit==2.0.1
pytz==2022.6
requests==2.28.1
urllib3==1.26.12