diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..19dd7de --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.analysis.typeCheckingMode": "basic" +} diff --git a/README.md b/README.md index 602d685..3e46125 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,28 @@ Reverse GeoLocate from XMP sidecar files with optional LightRoom DB read This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database. -#### Installing and setting up +## Development Setup + +```sh +python3 -m venv .venv +source .venv/bin/activate +.venv/bin/python -m pip install -U pip setuptools wheel +``` + +Then install the requests and python-xmp-toolkit modules from below + +## Installing and setting up The script uses the following external non defauly python libraries + * xmp toolkit * requests install both with the pip3 command -``` -pip3 install requests -pip3 install python-xmp-toolkit + +```sh +pip install requests +pip install python-xmp-toolkit ``` XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly. diff --git a/bin/reverse_geolocate.py b/bin/reverse_geolocate.py index 361d94c..4447c94 100755 --- a/bin/reverse_geolocate.py +++ b/bin/reverse_geolocate.py @@ -3,15 +3,14 @@ # AUTHOR : Clemens Schwaighofer # DATE : 2018/2/20 # LICENSE: GPLv3 -# DESC : Set the reverse Geo location (name) from Lat/Long data in XMP files in a lightroom catalogue -# * tries to get pre-set geo location from LR catalog -# * if not found tries to get data from Google -# * all data is translated into English with long vowl system (aka ou or oo is ō) +# DESC : +# Set the reverse Geo location (name) from Lat/Long data in XMP files +# in a lightroom catalogue +# * tries to get pre-set geo location from LR catalog +# * if not found tries to get data from Google +# * all data is translated into English with long vowl system (aka ou or oo is ō) # MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/) -import argparse -import sqlite3 -import requests import configparser import unicodedata # import textwrap @@ -19,10 +18,13 @@ import glob import os import sys import re -# Note XMPFiles does not work with sidecar files, need to read via XMPMeta -from libxmp import XMPMeta, consts +import argparse +import sqlite3 from shutil import copyfile, get_terminal_size from math import ceil, radians, sin, cos, atan2, sqrt +import requests +# Note XMPFiles does not work with sidecar files, need to read via XMPMeta +from libxmp import XMPMeta, consts ############################################################## # FUNCTIONS @@ -32,13 +34,16 @@ from math import ceil, radians, sin, cos, atan2, sqrt # this is used by isLatin and onlyLatinChars cache_latin_letters = {} - # ARGPARSE HELPERS -# call: writable_dir_folder -# checks if this is a writeable folder OR file -# AND it works on nargs * -class writable_dir_folder(argparse.Action): +class WritableDirFolder(argparse.Action): + """ + checks if this is a writeable folder OR file + AND it works on nargs * + + Args: + argparse (_type_): _description_ + """ def __call__(self, parser, namespace, values, option_string=None): # we loop through list (this is because of nargs *) for prospective_dir in values: @@ -632,743 +637,790 @@ def getBackupFileCounter(xmp_file): return bk_file_counter ############################################################## -# ARGUMENT PARSNING +# ARGUMENT PARSING ############################################################## +def argument_parser(): + """ + Parses the command line arguments -parser = argparse.ArgumentParser( - description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files', - # formatter_class=argparse.RawDescriptionHelpFormatter, - epilog='Sample: (todo)' -) + Returns: + Namespace: parsed arguments + """ -# xmp folder (or folders), or file (or files) -# note that the target directory or file needs to be writeable -parser.add_argument( - '-i', - '--include-source', - required=True, - nargs='*', - action=writable_dir_folder, - dest='xmp_sources', - metavar='XMP SOURCE FOLDER', - help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here' -) -# exclude folders -parser.add_argument( - '-x', - '--exclude-source', - nargs='*', - action=writable_dir_folder, - dest='exclude_sources', - metavar='EXCLUDE XMP SOURCE FOLDER', - help='Folders and files that will be excluded.' -) + parser = argparse.ArgumentParser( + description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files', + # formatter_class=argparse.RawDescriptionHelpFormatter, + epilog='Sample: (todo)' + ) -# LR database (base folder) -# get .lrcat file in this folder -parser.add_argument( - '-l', - '--lightroom', - # required=True, - action=readable_dir, - dest='lightroom_folder', - metavar='LIGHTROOM FOLDER', - help='Lightroom catalogue base folder' -) + # xmp folder (or folders), or file (or files) + # note that the target directory or file needs to be writeable + parser.add_argument( + '-i', + '--include-source', + required=True, + nargs='*', + action=WritableDirFolder, + dest='xmp_sources', + metavar='XMP SOURCE FOLDER', + help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here' + ) + # exclude folders + parser.add_argument( + '-x', + '--exclude-source', + nargs='*', + action=WritableDirFolder, + dest='exclude_sources', + metavar='EXCLUDE XMP SOURCE FOLDER', + help='Folders and files that will be excluded.' + ) -# strict LR check with base path next to the file base name -parser.add_argument( - '-s', - '--strict', - dest='lightroom_strict', - action='store_true', - help='Do strict check for Lightroom files including Path in query' -) + # LR database (base folder) + # get .lrcat file in this folder + parser.add_argument( + '-l', + '--lightroom', + # required=True, + action=readable_dir, + dest='lightroom_folder', + metavar='LIGHTROOM FOLDER', + help='Lightroom catalogue base folder' + ) -# set behaviour override -# FLAG: default: only set not filled -# other: overwrite all or overwrite if one is missing, overwrite specifc field (as defined below) -# fields: Location, City, State, Country, CountryCode -parser.add_argument( - '-f', - '--field', - action='append', - type=str.lower, # make it lowercase for check - choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'], - dest='field_controls', - metavar='', - help='On default only set fields that are not set yet. Options are: '\ - 'Overwrite (write all new), Location, City, State, Country, CountryCode. '\ - 'Multiple can be given for combination overwrite certain fields only or set only certain fields. '\ - 'If with overwrite the field will be overwritten if already set, else it will be always skipped.' -) + # strict LR check with base path next to the file base name + parser.add_argument( + '-s', + '--strict', + dest='lightroom_strict', + action='store_true', + help='Do strict check for Lightroom files including Path in query' + ) -parser.add_argument( - '-d', - '--fuzzy-cache', - type=str.lower, - action=distance_values, - nargs='?', - const='10m', # default is 10m - dest='fuzzy_distance', - metavar='FUZZY DISTANCE', - help='Allow fuzzy distance cache lookup. Optional distance can be given, '\ - 'if not set default of 10m is used. '\ - 'Allowed argument is in the format of 12m or 12km' -) + # set behaviour override + # FLAG: default: only set not filled + # other: overwrite all or overwrite if one is missing, + # overwrite specifc field (as defined below) + # fields: Location, City, State, Country, CountryCode + parser.add_argument( + '-f', + '--field', + action='append', + type=str.lower, # make it lowercase for check + choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'], + dest='field_controls', + metavar='', + help=( + 'On default only set fields that are not set yet. Options are: ' + 'Overwrite (write all new), Location, City, State, Country, CountryCode. ' + 'Multiple can be given for combination overwrite certain fields only ' + 'or set only certain fields. ' + 'If with overwrite the field will be overwritten if already set, ' + 'else it will be always skipped.' + ) + ) -# Google Maps API key to overcome restrictions -parser.add_argument( - '-g', - '--google', - dest='google_api_key', - metavar='GOOGLE API KEY', - help='Set a Google API Maps key to overcome the default lookup limitations' -) + parser.add_argument( + '-d', + '--fuzzy-cache', + type=str.lower, + action=distance_values, + nargs='?', + const='10m', # default is 10m + dest='fuzzy_distance', + metavar='FUZZY DISTANCE', + help=( + 'Allow fuzzy distance cache lookup. Optional distance can be given, ' + 'if not set default of 10m is used. ' + 'Allowed argument is in the format of 12m or 12km' + ) + ) -# use open street maps -parser.add_argument( - '-o', - '--openstreetmap', - dest='use_openstreetmap', - action='store_true', - help='Use openstreetmap instead of Google' -) + # Google Maps API key to overcome restrictions + parser.add_argument( + '-g', + '--google', + dest='google_api_key', + metavar='GOOGLE API KEY', + help='Set a Google API Maps key to overcome the default lookup limitations' + ) -# email of open street maps requests -parser.add_argument( - '-e', - '--email', - dest='email', - metavar='EMIL ADDRESS', - help='An email address for OpenStreetMap' -) + # use open street maps + parser.add_argument( + '-o', + '--openstreetmap', + dest='use_openstreetmap', + action='store_true', + help='Use openstreetmap instead of Google' + ) -# write api/email settings to config file -parser.add_argument( - '-w', - '--write-settings', - dest='config_write', - action='store_true', - help='Write Google API or OpenStreetMap email to config file' -) + # email of open street maps requests + parser.add_argument( + '-e', + '--email', + dest='email', + metavar='EMIL ADDRESS', + help='An email address for OpenStreetMap' + ) -# only read data and print on screen, do not write anything -parser.add_argument( - '-r', - '--read-only', - dest='read_only', - action='store_true', - help='Read current values from the XMP file only, do not read from LR or lookup any data and write back' -) + # write api/email settings to config file + parser.add_argument( + '-w', + '--write-settings', + dest='config_write', + action='store_true', + help='Write Google API or OpenStreetMap email to config file' + ) -# only list unset ones -parser.add_argument( - '-u', - '--unset-only', - dest='unset_only', - action='store_true', - help='Only list unset XMP files' -) + # only read data and print on screen, do not write anything + parser.add_argument( + '-r', + '--read-only', + dest='read_only', + action='store_true', + help=( + 'Read current values from the XMP file only, ' + 'do not read from LR or lookup any data and write back' + ) + ) -# only list unset GPS codes -parser.add_argument( - '-p', - '--unset-gps-only', - dest='unset_gps_only', - action='store_true', - help='Only list unset XMP files for GPS fields' -) + # only list unset ones + parser.add_argument( + '-u', + '--unset-only', + dest='unset_only', + action='store_true', + help='Only list unset XMP files' + ) -# don't try to do auto adjust in list view -parser.add_argument( - '-a', - '--no-autoadjust', - dest='no_autoadjust', - action='store_true', - help='Don\'t try to auto adjust columns' -) + # only list unset GPS codes + parser.add_argument( + '-p', + '--unset-gps-only', + dest='unset_gps_only', + action='store_true', + help='Only list unset XMP files for GPS fields' + ) -# compact view, compresses columns down to a minimum -parser.add_argument( - '-c', - '--compact', - dest='compact_view', - action='store_true', - help='Very compact list view' -) + # don't try to do auto adjust in list view + parser.add_argument( + '-a', + '--no-autoadjust', + dest='no_autoadjust', + action='store_true', + help='Don\'t try to auto adjust columns' + ) -# Do not create backup files -parser.add_argument( - '-n', - '--nobackup', - dest='no_xmp_backup', - action='store_true', - help='Do not create a backup from the XMP file' -) + # compact view, compresses columns down to a minimum + parser.add_argument( + '-c', + '--compact', + dest='compact_view', + action='store_true', + help='Very compact list view' + ) -# verbose args for more detailed output -parser.add_argument( - '-v', - '--verbose', - action='count', - dest='verbose', - help='Set verbose output level' -) + # Do not create backup files + parser.add_argument( + '-n', + '--nobackup', + dest='no_xmp_backup', + action='store_true', + help='Do not create a backup from the XMP file' + ) -# debug flag -parser.add_argument('--debug', action='store_true', dest='debug', help='Set detailed debug output') -# test flag -parser.add_argument('--test', action='store_true', dest='test', help='Do not write data back to file') + # verbose args for more detailed output + parser.add_argument( + '-v', + '--verbose', + action='count', + dest='verbose', + help='Set verbose output level' + ) -# read in the argumens -args = parser.parse_args() + # debug flag + parser.add_argument( + '--debug', action='store_true', dest='debug', help='Set detailed debug output' + ) + # test flag + parser.add_argument( + '--test', action='store_true', dest='test', help='Do not write data back to file' + ) + + # read in the argumens + return parser.parse_args() ############################################################## # MAIN CODE ############################################################## -# init verbose to 0 if not set -if not args.verbose: - args.verbose = 0 -# init exclude source to list if not set -if not args.exclude_sources: - args.exclude_sources = [] -# init args unset (for list view) with 0 if unset -if not args.unset_only: - args.unset_only = 0 +def main(): + """ + Main code run + """ + args = argument_parser() -if args.debug: - print("### ARGUMENT VARS: I: {incl}, X: {excl}, L: {lr}, F: {fc}, D: {fdist}, M: {osm}, G: {gp}, E: {em}, R: {read}, U: {us}, A: {adj}, C: {cmp}, N: {nbk}, W: {wrc}, V: {v}, D: {d}, T: {t}".format( - incl=args.xmp_sources, - excl=args.exclude_sources, - lr=args.lightroom_folder, - fc=args.field_controls, - fdist=args.fuzzy_distance, - osm=args.use_openstreetmap, - gp=args.google_api_key, - em=args.email, - read=args.read_only, - us=args.unset_only, - adj=args.no_autoadjust, - cmp=args.compact_view, - nbk=args.no_xmp_backup, - wrc=args.config_write, - v=args.verbose, - d=args.debug, - t=args.test - )) + # init verbose to 0 if not set + if not args.verbose: + args.verbose = 0 + # init exclude source to list if not set + if not args.exclude_sources: + args.exclude_sources = [] + # init args unset (for list view) with 0 if unset + if not args.unset_only: + args.unset_only = 0 -# error flag -error = False -# set search map type -map_type = 'google' if not args.use_openstreetmap else 'openstreetmap' -# if -g and -o, error -if args.google_api_key and args.use_openstreetmap: - print("You cannot set a Google API key and use OpenStreetMap at the same time") - error = True -# or if -g and -e -if args.google_api_key and args.email: - print("You cannot set a Google API key and OpenStreetMap email at the same time") - error = True -# or -e and no -o -if args.email and not args.use_openstreetmap: - print("You cannot set an OpenStreetMap email and not use OpenStreetMap") - error = True -# if email and not basic valid email (@ .) -if args.email: - if not re.match(r'^.+@.+\.[A-Za-z]{1,}$', args.email): - print("Not a valid email for OpenStreetMap: {}".format(args.email)) - error = True -# on error exit here -if error: - sys.exit(1) - -config = configparser.ConfigParser() -# try to find config file in following order -# $HOME/.config/ -config_file = 'reverse_geolocate.cfg' -config_folder = os.path.expanduser('~/.config/reverseGeolocate/') -config_data = '{}{}'.format(config_folder, config_file) -# if file exists read, if not skip unless we have write flag and google api or openstreetmaps email -if os.path.isfile(config_data): - config.read(config_data) - # check if api group & setting is there. also never overwrite argument given data - if 'API' in config: - if 'googleapikey' in config['API']: - if not args.google_api_key: - args.google_api_key = config['API']['googleapikey'] - if 'openstreetmapemail' in config['API']: - if not args.email: - args.email = config['API']['openstreetmapemail'] -# write data if exists and changed -if args.config_write and (args.google_api_key or args.email): - config_change = False - # check if new value differs, if yes, change and write - if 'API' not in config: - config['API'] = {} - if args.google_api_key and ('googleapikey' not in config['API'] or config['API']['googleapikey'] != args.google_api_key): - config['API']['googleapikey'] = args.google_api_key - config_change = True - if args.email and ('openstreetmapemail' not in config['API'] or config['API']['openstreetmapemail'] != args.email): - config['API']['openstreetmapemail'] = args.email - config_change = True - if config_change: - # if we do not have the base folder create that first - if not os.path.exists(config_folder): - os.makedirs(config_folder) - with open(config_data, 'w') as fptr: - config.write(fptr) -if args.debug: - print("### OVERRIDE API: G: {}, O: {}".format(args.google_api_key, args.email)) - -# The XMP fields const lookup values -# XML/XMP -# READ: -# exif:GPSLatitude -# exif:GPSLongitude -# READ for if filled -# Iptc4xmpCore:Location -# photoshop:City -# photoshop:State -# photoshop:Country -# Iptc4xmpCore:CountryCode -xmp_fields = { - 'GPSLatitude': consts.XMP_NS_EXIF, # EXIF GPSLat/Long are stored in Degree,Min.Sec[NESW] format - 'GPSLongitude': consts.XMP_NS_EXIF, - 'Location': consts.XMP_NS_IPTCCore, - 'City': consts.XMP_NS_Photoshop, - 'State': consts.XMP_NS_Photoshop, - 'Country': consts.XMP_NS_Photoshop, - 'CountryCode': consts.XMP_NS_IPTCCore -} -# non lat/long fields (for loc loops) -data_set_loc = ('Location', 'City', 'State', 'Country', 'CountryCode') -# one xmp data set -data_set = { - 'GPSLatitude': '', - 'GPSLongitude': '', - 'Location': '', - 'City': '', - 'State': '', - 'Country': '', - 'CountryCode': '' -} -# original set for compare (is constant unchanged) -data_set_original = {} -# cache set to avoid double lookups for identical Lat/Ling -data_cache = {} -# work files, all files + folders we need to work on -work_files = [] -# all failed files -failed_files = [] -# use lightroom -use_lightroom = False -# cursors & query -query = '' -cur = '' -# count variables -count = { - 'all': 0, - 'listed': 0, - 'read': 0, - 'map': 0, - 'cache': 0, - 'fuzzy_cache': 0, - 'lightroom': 0, - 'changed': 0, - 'failed': 0, - 'skipped': 0, - 'not_found': 0, - 'many_found': 0, -} - -# do lightroom stuff only if we have the lightroom folder -if args.lightroom_folder: - # query string for lightroom DB check - query = 'SELECT Adobe_images.id_local, AgLibraryFile.baseName, AgLibraryRootFolder.absolutePath, AgLibraryRootFolder.name as realtivePath, AgLibraryFolder.pathFromRoot, AgLibraryFile.originalFilename, ' - query += 'AgHarvestedExifMetadata.gpsLatitude, AgHarvestedExifMetadata.gpsLongitude, ' - query += 'AgHarvestedIptcMetadata.locationDataOrigination, AgInternedIptcLocation.value as Location, AgInternedIptcCity.value as City, ' - query += 'AgInternedIptcState.value as State, AgInternedIptcCountry.value as Country, AgInternedIptcIsoCountryCode.value as CountryCode ' - query += 'FROM AgLibraryFile, AgHarvestedExifMetadata, AgLibraryFolder, AgLibraryRootFolder, Adobe_images ' - query += 'LEFT JOIN AgHarvestedIptcMetadata ON Adobe_images.id_local = AgHarvestedIptcMetadata.image ' - query += 'LEFT JOIN AgInternedIptcLocation ON AgHarvestedIptcMetadata.locationRef = AgInternedIptcLocation.id_local ' - query += 'LEFT JOIN AgInternedIptcCity ON AgHarvestedIptcMetadata.cityRef = AgInternedIptcCity.id_local ' - query += 'LEFT JOIN AgInternedIptcState ON AgHarvestedIptcMetadata.stateRef = AgInternedIptcState.id_local ' - query += 'LEFT JOIN AgInternedIptcCountry ON AgHarvestedIptcMetadata.countryRef = AgInternedIptcCountry.id_local ' - query += 'LEFT JOIN AgInternedIptcIsoCountryCode ON AgHarvestedIptcMetadata.isoCountryCodeRef = AgInternedIptcIsoCountryCode.id_local ' - query += 'WHERE Adobe_images.rootFile = AgLibraryFile.id_local AND Adobe_images.id_local = AgHarvestedExifMetadata.image AND AgLibraryFile.folder = AgLibraryFolder.id_local AND AgLibraryFolder.rootFolder = AgLibraryRootFolder.id_local ' - query += 'AND AgLibraryFile.baseName = ?' - # absolutePath + pathFromRoot = path of XMP file - XMP file - if args.lightroom_strict: - query += 'AND AgLibraryRootFolder.absolutePath || AgLibraryFolder.pathFromRoot = ?' - - # connect to LR database for reading - # open the folder and look for the first lrcat file in there - for file in os.listdir(args.lightroom_folder): - if file.endswith('.lrcat'): - lightroom_database = os.path.join(args.lightroom_folder, file) - lrdb = sqlite3.connect(lightroom_database) - if not lightroom_database or not lrdb: - print("(!) We could not find a lrcat file in the given lightroom folder or DB connection failed: {}".format(args.lightroom_folder)) - # flag for end - error = True - else: - # set row so we can access each element by the name - lrdb.row_factory = sqlite3.Row - # set cursor - cur = lrdb.cursor() - # flag that we have Lightroom DB - use_lightroom = True if args.debug: - print("### USE Lightroom {}".format(use_lightroom)) - -# on error exit here -if error: - sys.exit(1) - -# init the XML meta for handling -xmp = XMPMeta() - -# loop through the xmp_sources (folder or files) and read in the XMP data for LAT/LONG, other data -for xmp_file_source in args.xmp_sources: - # if folder, open and loop - # NOTE: we do check for folders in there, if there are we recourse traverse them - # also check that folder is not in exclude list - if os.path.isdir(xmp_file_source) and xmp_file_source.rstrip('/') not in [x.rstrip('/') for x in args.exclude_sources]: - # open folder and look for any .xmp files and push them into holding array - # if there are folders, dive into them - # or glob glob all .xmp files + directory - for root, dirs, files in os.walk(xmp_file_source): - for file in sorted(files): - # 1) but has no .BK. inside - # 2) file is not in exclude list - # 3) full folder is not in exclude list - if file.endswith(".xmp") and ".BK." not in file \ - and "{}/{}".format(root, file) not in args.exclude_sources \ - and root.rstrip('/') not in [x.rstrip('/') for x in args.exclude_sources]: - if "{}/{}".format(root, file) not in work_files: - work_files.append("{}/{}".format(root, file)) - count['all'] += 1 - else: - # not already added to list and not in the exclude list either - if xmp_file_source not in work_files and xmp_file_source not in args.exclude_sources: - work_files.append(xmp_file_source) - count['all'] += 1 -if args.debug: - print("### Work Files {}".format(work_files)) - -# if we have read only we print list format style -if args.read_only: - # adjust the output width for the list view - format_length = outputListWidthAdjust() - - # after how many lines do we reprint the header - header_repeat = 50 - # how many pages will we have - page_all = ceil(len(work_files) / header_repeat) - # current page number - page_no = 1 - # the formatted line for the output - # 4 {} => final replace: data (2 pre replaces) - # 1 {} => length replace here - format_line = " {{{{filename:<{}}}}} | {{{{latitude:>{}}}}} | {{{{longitude:>{}}}}} | {{{{code:<{}}}}} | {{{{country:<{}}}}} | {{{{state:<{}}}}} | {{{{city:<{}}}}} | {{{{location:<{}}}}} | {{{{path:<{}}}}}".format( - "{filenamelen}", - format_length['latitude'], - format_length['longitude'], - format_length['code'], - "{countrylen}", - "{statelen}", - "{citylen}", - "{locationlen}", - "{pathlen}" # set path len replacer variable - ) - # header line format: - # blank line - # header title - # seperator line - header_line = '''{} -{} -{}'''.format( - '> Page {page_no:,}/{page_all:,}', # can later be set to something else, eg page numbers - # pre replace path length before we add the header titles - format_line.format( - filenamelen=format_length['filename'], - countrylen=format_length['country'], - statelen=format_length['state'], - citylen=format_length['city'], - locationlen=format_length['location'], - pathlen=format_length['path'] - ).format( # the header title line - filename='File'[:format_length['filename']], - latitude='Latitude'[:format_length['latitude']], - longitude='Longitude'[:format_length['longitude']], - code='Code', - country='Country'[:format_length['country']], - state='State'[:format_length['state']], - city='City'[:format_length['city']], - location='Location'[:format_length['location']], - path='Path'[:format_length['path']] - ), - "{}+{}+{}+{}+{}+{}+{}+{}+{}".format( # the header seperator line - '-' * (format_length['filename'] + 2), - '-' * (format_length['latitude'] + 2), - '-' * (format_length['longitude'] + 2), - '-' * (format_length['code'] + 2), - '-' * (format_length['country'] + 2), - '-' * (format_length['state'] + 2), - '-' * (format_length['city'] + 2), - '-' * (format_length['location'] + 2), - '-' * (format_length['path'] + 2) + print( + "### ARGUMENT VARS: " + f"I: {args.xmp_sources}, X: {args.exclude_sources}, L: {args.lightroom_folder}, " + f"F: {args.field_controls}, D: {args.fuzzy_distance}, M: {args.use_openstreetmap}, " + f"G: {args.google_api_key}, E: {args.email}, R: {args.read_only}, U: {args.unset_only}, " + f"A: {args.no_autoadjust}, C: {args.compact_view}, N: {args.no_xmp_backup}, " + f"W: {args.config_write}, V: {args.verbose}, D: {args.debug}, T: {args.test}" ) - ) - # print header - printHeader(header_line.format(page_no=page_no, page_all=page_all)) - # print no files found if we have no files - if not work_files: - print("{:<60}".format('[!!!] No files found')) -# ### MAIN WORK LOOP -# now we just loop through each file and work on them -for xmp_file in work_files: # noqa: C901 - if not args.read_only: - print("---> {}: ".format(xmp_file), end='') + # error flag + error = False + # set search map type + map_type = 'google' if not args.use_openstreetmap else 'openstreetmap' + # if -g and -o, error + if args.google_api_key and args.use_openstreetmap: + print("You cannot set a Google API key and use OpenStreetMap at the same time") + error = True + # or if -g and -e + if args.google_api_key and args.email: + print("You cannot set a Google API key and OpenStreetMap email at the same time") + error = True + # or -e and no -o + if args.email and not args.use_openstreetmap: + print("You cannot set an OpenStreetMap email and not use OpenStreetMap") + error = True + # if email and not basic valid email (@ .) + if args.email: + if not re.match(r'^.+@.+\.[A-Za-z]{1,}$', args.email): + print("Not a valid email for OpenStreetMap: {}".format(args.email)) + error = True + # on error exit here + if error: + sys.exit(1) - # ### ACTION FLAGs - write_file = False + config = configparser.ConfigParser() + # try to find config file in following order + # $HOME/.config/ + config_file = 'reverse_geolocate.cfg' + config_folder = os.path.expanduser('~/.config/reverseGeolocate/') + config_data = '{}{}'.format(config_folder, config_file) + # if file exists read, if not skip unless we have write flag and google api or openstreetmaps email + if os.path.isfile(config_data): + config.read(config_data) + # check if api group & setting is there. also never overwrite argument given data + if 'API' in config: + if 'googleapikey' in config['API']: + if not args.google_api_key: + args.google_api_key = config['API']['googleapikey'] + if 'openstreetmapemail' in config['API']: + if not args.email: + args.email = config['API']['openstreetmapemail'] + # write data if exists and changed + if args.config_write and (args.google_api_key or args.email): + config_change = False + # check if new value differs, if yes, change and write + if 'API' not in config: + config['API'] = {} + if args.google_api_key and ('googleapikey' not in config['API'] or config['API']['googleapikey'] != args.google_api_key): + config['API']['googleapikey'] = args.google_api_key + config_change = True + if args.email and ('openstreetmapemail' not in config['API'] or config['API']['openstreetmapemail'] != args.email): + config['API']['openstreetmapemail'] = args.email + config_change = True + if config_change: + # if we do not have the base folder create that first + if not os.path.exists(config_folder): + os.makedirs(config_folder) + with open(config_data, 'w') as fptr: + config.write(fptr) + if args.debug: + print("### OVERRIDE API: G: {}, O: {}".format(args.google_api_key, args.email)) - # ### XMP FILE READING - # open file & read all into buffer - with open(xmp_file, 'r') as fptr: - strbuffer = fptr.read() - # read fields from the XMP file and store in hash - xmp.parse_from_str(strbuffer) - for xmp_field in xmp_fields: - # need to check if propert exist or it will the exempi routine will fail - if xmp.does_property_exist(xmp_fields[xmp_field], xmp_field): - data_set[xmp_field] = xmp.get_property(xmp_fields[xmp_field], xmp_field) + # The XMP fields const lookup values + # XML/XMP + # READ: + # exif:GPSLatitude + # exif:GPSLongitude + # READ for if filled + # Iptc4xmpCore:Location + # photoshop:City + # photoshop:State + # photoshop:Country + # Iptc4xmpCore:CountryCode + xmp_fields = { + 'GPSLatitude': consts.XMP_NS_EXIF, # EXIF GPSLat/Long are stored in Degree,Min.Sec[NESW] format + 'GPSLongitude': consts.XMP_NS_EXIF, + 'Location': consts.XMP_NS_IPTCCore, + 'City': consts.XMP_NS_Photoshop, + 'State': consts.XMP_NS_Photoshop, + 'Country': consts.XMP_NS_Photoshop, + 'CountryCode': consts.XMP_NS_IPTCCore + } + # non lat/long fields (for loc loops) + data_set_loc = ('Location', 'City', 'State', 'Country', 'CountryCode') + # one xmp data set + data_set = { + 'GPSLatitude': '', + 'GPSLongitude': '', + 'Location': '', + 'City': '', + 'State': '', + 'Country': '', + 'CountryCode': '' + } + # original set for compare (is constant unchanged) + data_set_original = {} + # cache set to avoid double lookups for identical Lat/Ling + data_cache = {} + # work files, all files + folders we need to work on + work_files = [] + # all failed files + failed_files = [] + # use lightroom + use_lightroom = False + # cursors & query + query = '' + cur = '' + # count variables + count = { + 'all': 0, + 'listed': 0, + 'read': 0, + 'map': 0, + 'cache': 0, + 'fuzzy_cache': 0, + 'lightroom': 0, + 'changed': 0, + 'failed': 0, + 'skipped': 0, + 'not_found': 0, + 'many_found': 0, + } + + # do lightroom stuff only if we have the lightroom folder + if args.lightroom_folder: + # query string for lightroom DB check + query = 'SELECT Adobe_images.id_local, AgLibraryFile.baseName, AgLibraryRootFolder.absolutePath, AgLibraryRootFolder.name as realtivePath, AgLibraryFolder.pathFromRoot, AgLibraryFile.originalFilename, ' + query += 'AgHarvestedExifMetadata.gpsLatitude, AgHarvestedExifMetadata.gpsLongitude, ' + query += 'AgHarvestedIptcMetadata.locationDataOrigination, AgInternedIptcLocation.value as Location, AgInternedIptcCity.value as City, ' + query += 'AgInternedIptcState.value as State, AgInternedIptcCountry.value as Country, AgInternedIptcIsoCountryCode.value as CountryCode ' + query += 'FROM AgLibraryFile, AgHarvestedExifMetadata, AgLibraryFolder, AgLibraryRootFolder, Adobe_images ' + query += 'LEFT JOIN AgHarvestedIptcMetadata ON Adobe_images.id_local = AgHarvestedIptcMetadata.image ' + query += 'LEFT JOIN AgInternedIptcLocation ON AgHarvestedIptcMetadata.locationRef = AgInternedIptcLocation.id_local ' + query += 'LEFT JOIN AgInternedIptcCity ON AgHarvestedIptcMetadata.cityRef = AgInternedIptcCity.id_local ' + query += 'LEFT JOIN AgInternedIptcState ON AgHarvestedIptcMetadata.stateRef = AgInternedIptcState.id_local ' + query += 'LEFT JOIN AgInternedIptcCountry ON AgHarvestedIptcMetadata.countryRef = AgInternedIptcCountry.id_local ' + query += 'LEFT JOIN AgInternedIptcIsoCountryCode ON AgHarvestedIptcMetadata.isoCountryCodeRef = AgInternedIptcIsoCountryCode.id_local ' + query += 'WHERE Adobe_images.rootFile = AgLibraryFile.id_local AND Adobe_images.id_local = AgHarvestedExifMetadata.image AND AgLibraryFile.folder = AgLibraryFolder.id_local AND AgLibraryFolder.rootFolder = AgLibraryRootFolder.id_local ' + query += 'AND AgLibraryFile.baseName = ?' + # absolutePath + pathFromRoot = path of XMP file - XMP file + if args.lightroom_strict: + query += 'AND AgLibraryRootFolder.absolutePath || AgLibraryFolder.pathFromRoot = ?' + + # connect to LR database for reading + # open the folder and look for the first lrcat file in there + for file in os.listdir(args.lightroom_folder): + if file.endswith('.lrcat'): + lightroom_database = os.path.join(args.lightroom_folder, file) + lrdb = sqlite3.connect(lightroom_database) + if not lightroom_database or not lrdb: + print("(!) We could not find a lrcat file in the given lightroom folder or DB connection failed: {}".format(args.lightroom_folder)) + # flag for end + error = True else: - data_set[xmp_field] = '' + # set row so we can access each element by the name + lrdb.row_factory = sqlite3.Row + # set cursor + cur = lrdb.cursor() + # flag that we have Lightroom DB + use_lightroom = True if args.debug: - print("### => XMP: {}:{} => {}".format(xmp_fields[xmp_field], xmp_field, data_set[xmp_field])) + print("### USE Lightroom {}".format(use_lightroom)) + + # on error exit here + if error: + sys.exit(1) + + # init the XML meta for handling + xmp = XMPMeta() + + # loop through the xmp_sources (folder or files) and read in the XMP data for LAT/LONG, other data + for xmp_file_source in args.xmp_sources: + # if folder, open and loop + # NOTE: we do check for folders in there, if there are we recourse traverse them + # also check that folder is not in exclude list + if os.path.isdir(xmp_file_source) and xmp_file_source.rstrip('/') not in [x.rstrip('/') for x in args.exclude_sources]: + # open folder and look for any .xmp files and push them into holding array + # if there are folders, dive into them + # or glob glob all .xmp files + directory + for root, dirs, files in os.walk(xmp_file_source): + for file in sorted(files): + # 1) but has no .BK. inside + # 2) file is not in exclude list + # 3) full folder is not in exclude list + if file.endswith(".xmp") and ".BK." not in file \ + and "{}/{}".format(root, file) not in args.exclude_sources \ + and root.rstrip('/') not in [x.rstrip('/') for x in args.exclude_sources]: + if "{}/{}".format(root, file) not in work_files: + work_files.append("{}/{}".format(root, file)) + count['all'] += 1 + else: + # not already added to list and not in the exclude list either + if xmp_file_source not in work_files and xmp_file_source not in args.exclude_sources: + work_files.append(xmp_file_source) + count['all'] += 1 + if args.debug: + print("### Work Files {}".format(work_files)) + + # if we have read only we print list format style if args.read_only: - # view only if list all or if data is unset - if (not args.unset_only and not args.unset_gps_only) or (args.unset_only and '' in data_set.values()) or (args.unset_gps_only and (not data_set['GPSLatitude'] or not data_set['GPSLongitude'])): - # for read only we print out the data formatted - # headline check, do we need to print that - count['read'] = printHeader(header_line.format(page_no=page_no, page_all=page_all), count['read'], header_repeat) - # the data content - print(format_line.format( - # for all possible non latin fields we do adjust if it has double byte characters inside - filenamelen=formatLen(shortenPath(xmp_file, format_length['filename'], file_only=True), format_length['filename']), - countrylen=formatLen(shortenString(data_set['Country'], width=format_length['country']), format_length['country']), - statelen=formatLen(shortenString(data_set['State'], width=format_length['state']), format_length['state']), - citylen=formatLen(shortenString(data_set['City'], width=format_length['city']), format_length['city']), - locationlen=formatLen(shortenString(data_set['Location'], width=format_length['location']), format_length['location']), - pathlen=formatLen(shortenPath(xmp_file, format_length['path'], path_only=True), format_length['path']) - ).format( - filename=shortenPath(xmp_file, format_length['filename'], file_only=True), # shorten from the left - latitude=str(convertDMStoLat(data_set['GPSLatitude']))[:format_length['latitude']], # cut off from the right - longitude=str(convertDMStoLong(data_set['GPSLongitude']))[:format_length['longitude']], - code=data_set['CountryCode'][:2].center(4), # is only 2 chars - country=shortenString(data_set['Country'], width=format_length['country']), # shorten from the right - state=shortenString(data_set['State'], width=format_length['state']), - city=shortenString(data_set['City'], width=format_length['city']), - location=shortenString(data_set['Location'], width=format_length['location']), - path=shortenPath(xmp_file, format_length['path'], path_only=True) - ) + # adjust the output width for the list view + format_length = outputListWidthAdjust() + + # after how many lines do we reprint the header + header_repeat = 50 + # how many pages will we have + page_all = ceil(len(work_files) / header_repeat) + # current page number + page_no = 1 + # the formatted line for the output + # 4 {} => final replace: data (2 pre replaces) + # 1 {} => length replace here + format_line = " {{{{filename:<{}}}}} | {{{{latitude:>{}}}}} | {{{{longitude:>{}}}}} | {{{{code:<{}}}}} | {{{{country:<{}}}}} | {{{{state:<{}}}}} | {{{{city:<{}}}}} | {{{{location:<{}}}}} | {{{{path:<{}}}}}".format( + "{filenamelen}", + format_length['latitude'], + format_length['longitude'], + format_length['code'], + "{countrylen}", + "{statelen}", + "{citylen}", + "{locationlen}", + "{pathlen}" # set path len replacer variable + ) + # header line format: + # blank line + # header title + # seperator line + header_line = '''{} + {} + {}'''.format( + '> Page {page_no:,}/{page_all:,}', # can later be set to something else, eg page numbers + # pre replace path length before we add the header titles + format_line.format( + filenamelen=format_length['filename'], + countrylen=format_length['country'], + statelen=format_length['state'], + citylen=format_length['city'], + locationlen=format_length['location'], + pathlen=format_length['path'] + ).format( # the header title line + filename='File'[:format_length['filename']], + latitude='Latitude'[:format_length['latitude']], + longitude='Longitude'[:format_length['longitude']], + code='Code', + country='Country'[:format_length['country']], + state='State'[:format_length['state']], + city='City'[:format_length['city']], + location='Location'[:format_length['location']], + path='Path'[:format_length['path']] + ), + "{}+{}+{}+{}+{}+{}+{}+{}+{}".format( # the header seperator line + '-' * (format_length['filename'] + 2), + '-' * (format_length['latitude'] + 2), + '-' * (format_length['longitude'] + 2), + '-' * (format_length['code'] + 2), + '-' * (format_length['country'] + 2), + '-' * (format_length['state'] + 2), + '-' * (format_length['city'] + 2), + '-' * (format_length['location'] + 2), + '-' * (format_length['path'] + 2) ) - count['listed'] += 1 - else: - # ### LR Action Flag (data ok) - lightroom_data_ok = True - # ### LIGHTROOM DB READING - # read in data from DB if we uave lightroom folder - if use_lightroom: - # get the base file name, we need this for lightroom - xmp_file_basename = os.path.splitext(os.path.split(xmp_file)[1])[0] - # try to get this file name from the DB - lr_query_params = [xmp_file_basename] - # for strict check we need to get the full path, and add / as the LR stores the last folder with / - if args.lightroom_strict: - xmp_file_path = "{}/{}".format(os.path.split(xmp_file)[0], '/') - lr_query_params.append(xmp_file_path) - cur.execute(query, lr_query_params) - # get the row data - lrdb_row = cur.fetchone() - # abort the read because we found more than one row - if cur.fetchone() is not None: - print("(!) Lightroom DB returned more than one more row") - lightroom_data_ok = False - count['many_found'] += 1 - # Notify if we couldn't find one - elif not lrdb_row: - print("(!) Could not get data from Lightroom DB") - lightroom_data_ok = False - count['not_found'] += 1 - if args.debug and lrdb_row: - print("### LightroomDB: {} / {}".format(tuple(lrdb_row), lrdb_row.keys())) + ) + # print header + printHeader(header_line.format(page_no=page_no, page_all=page_all)) + # print no files found if we have no files + if not work_files: + print("{:<60}".format('[!!!] No files found')) - # create a duplicate copy for later checking if something changed - data_set_original = data_set.copy() - # check if LR exists and use this to compare to XMP data - # is LR GPS and no XMP GPS => use LR and set XMP - # same for location names - # if missing in XMP but in LR -> set in XMP - # if missing in both do lookup in Maps - if use_lightroom and lightroom_data_ok: - # check lat/long separate - if lrdb_row['gpsLatitude'] and not data_set['GPSLatitude']: - # we need to convert to the Degree,Min.sec[NSEW] format - data_set['GPSLatitude'] = convertLatToDMS(lrdb_row['gpsLatitude']) - if lrdb_row['gpsLongitude'] and not data_set['GPSLongitude']: - data_set['GPSLongitude'] = convertLongToDMS(lrdb_row['gpsLongitude']) - # now check Location, City, etc - for loc in data_set_loc: - # overwrite original set (read from XMP) with LR data if original data is missing - if lrdb_row[loc] and not data_set[loc]: - data_set[loc] = lrdb_row[loc] - if args.debug: - print("### -> LR: {} => {}".format(loc, lrdb_row[loc])) - # base set done, now check if there is anything unset in the data_set, if yes do a lookup in maps - # run this through the overwrite checker to get unset if we have a forced overwrite - has_unset = False - failed = False - from_cache = False - for loc in data_set_loc: - if checkOverwrite(data_set[loc], loc, args.field_controls): - has_unset = True - if has_unset: - # check if lat/long is in cache - cache_key = '{}#{}'.format(data_set['GPSLongitude'], data_set['GPSLatitude']) - if args.debug: - print("### *** CACHE: {}: {}".format(cache_key, 'NO' if cache_key not in data_cache else 'YES')) - # main chache check = identical - # second cache level check is on distance: - # default distance is 10m, can be set via flag - # check distance to previous cache entries (reverse newest to oldest) and match before we do google lookup - if cache_key not in data_cache: - has_fuzzy_cache = False - if args.fuzzy_distance: - shortest_distance = args.fuzzy_distance - best_match_latlong = '' - # check if we have fuzzy distance, if no valid found do maps lookup - for _cache_key in data_cache: - # split up cache key so we can use in the distance calc method - to_lat_long = _cache_key.split('#') - # get the distance based on current set + cached set - # print("Lookup f-long {} f-lat {} t-long {} t-lat {}".format(data_set['GPSLongitude'], data_set['GPSLatitude'], to_lat_long[0], to_lat_long[1])) - distance = getDistance(from_longitude=data_set['GPSLongitude'], from_latitude=data_set['GPSLatitude'], to_longitude=to_lat_long[0], to_latitude=to_lat_long[1]) - if args.debug: - print("### **= FUZZY CACHE: => distance: {} (m), shortest: {}".format(distance, shortest_distance)) - if distance <= shortest_distance: - # set new distance and keep current best matching location - shortest_distance = distance - best_match_latlong = _cache_key - has_fuzzy_cache = True - if args.debug: - print("### ***= FUZZY CACHE: YES => Best match: {}".format(best_match_latlong)) - if not has_fuzzy_cache: - # get location from maps (google or openstreetmap) - maps_location = reverseGeolocate(latitude=data_set['GPSLatitude'], longitude=data_set['GPSLongitude'], map_type=map_type) - # cache data with Lat/Long - data_cache[cache_key] = maps_location - from_cache = False - else: - maps_location = data_cache[best_match_latlong] - # cache this one, because the next one will match this one too - # we don't need to loop search again for the same fuzzy location - data_cache[cache_key] = maps_location - count['cache'] += 1 - count['fuzzy_cache'] += 1 - from_cache = True + # ### MAIN WORK LOOP + # now we just loop through each file and work on them + for xmp_file in work_files: # noqa: C901 + if not args.read_only: + print("---> {}: ".format(xmp_file), end='') + + # ### ACTION FLAGs + write_file = False + + # ### XMP FILE READING + # open file & read all into buffer + with open(xmp_file, 'r') as fptr: + strbuffer = fptr.read() + # read fields from the XMP file and store in hash + xmp.parse_from_str(strbuffer) + for xmp_field in xmp_fields: + # need to check if propert exist or it will the exempi routine will fail + if xmp.does_property_exist(xmp_fields[xmp_field], xmp_field): + data_set[xmp_field] = xmp.get_property(xmp_fields[xmp_field], xmp_field) else: - # load location from cache - maps_location = data_cache[cache_key] - count['cache'] += 1 - from_cache = True - # overwrite sets (note options check here) + data_set[xmp_field] = '' if args.debug: - print("### Map Location ({}): {}".format(map_type, maps_location)) - # must have at least the country set to write anything back - if maps_location['Country']: - for loc in data_set_loc: - # only write to XMP if overwrite check passes - if checkOverwrite(data_set_original[loc], loc, args.field_controls): - data_set[loc] = maps_location[loc] - xmp.set_property(xmp_fields[loc], loc, maps_location[loc]) - write_file = True - if write_file: - count['map'] += 1 - else: - print("(!) Could not geo loaction data ", end='') - failed = True + print("### => XMP: {}:{} => {}".format(xmp_fields[xmp_field], xmp_field, data_set[xmp_field])) + if args.read_only: + # view only if list all or if data is unset + if (not args.unset_only and not args.unset_gps_only) or (args.unset_only and '' in data_set.values()) or (args.unset_gps_only and (not data_set['GPSLatitude'] or not data_set['GPSLongitude'])): + # for read only we print out the data formatted + # headline check, do we need to print that + count['read'] = printHeader(header_line.format(page_no=page_no, page_all=page_all), count['read'], header_repeat) + # the data content + print(format_line.format( + # for all possible non latin fields we do adjust if it has double byte characters inside + filenamelen=formatLen(shortenPath(xmp_file, format_length['filename'], file_only=True), format_length['filename']), + countrylen=formatLen(shortenString(data_set['Country'], width=format_length['country']), format_length['country']), + statelen=formatLen(shortenString(data_set['State'], width=format_length['state']), format_length['state']), + citylen=formatLen(shortenString(data_set['City'], width=format_length['city']), format_length['city']), + locationlen=formatLen(shortenString(data_set['Location'], width=format_length['location']), format_length['location']), + pathlen=formatLen(shortenPath(xmp_file, format_length['path'], path_only=True), format_length['path']) + ).format( + filename=shortenPath(xmp_file, format_length['filename'], file_only=True), # shorten from the left + latitude=str(convertDMStoLat(data_set['GPSLatitude']))[:format_length['latitude']], # cut off from the right + longitude=str(convertDMStoLong(data_set['GPSLongitude']))[:format_length['longitude']], + code=data_set['CountryCode'][:2].center(4), # is only 2 chars + country=shortenString(data_set['Country'], width=format_length['country']), # shorten from the right + state=shortenString(data_set['State'], width=format_length['state']), + city=shortenString(data_set['City'], width=format_length['city']), + location=shortenString(data_set['Location'], width=format_length['location']), + path=shortenPath(xmp_file, format_length['path'], path_only=True) + ) + ) + count['listed'] += 1 else: - if args.debug: - print("Lightroom data use: {}, Lightroom data ok: {}".format(use_lightroom, lightroom_data_ok)) - # check if the data_set differs from the original (LR db load) - # if yes write, else skip + # ### LR Action Flag (data ok) + lightroom_data_ok = True + # ### LIGHTROOM DB READING + # read in data from DB if we uave lightroom folder + if use_lightroom: + # get the base file name, we need this for lightroom + xmp_file_basename = os.path.splitext(os.path.split(xmp_file)[1])[0] + # try to get this file name from the DB + lr_query_params = [xmp_file_basename] + # for strict check we need to get the full path, and add / as the LR stores the last folder with / + if args.lightroom_strict: + xmp_file_path = "{}/{}".format(os.path.split(xmp_file)[0], '/') + lr_query_params.append(xmp_file_path) + cur.execute(query, lr_query_params) + # get the row data + lrdb_row = cur.fetchone() + # abort the read because we found more than one row + if cur.fetchone() is not None: + print("(!) Lightroom DB returned more than one more row") + lightroom_data_ok = False + count['many_found'] += 1 + # Notify if we couldn't find one + elif not lrdb_row: + print("(!) Could not get data from Lightroom DB") + lightroom_data_ok = False + count['not_found'] += 1 + if args.debug and lrdb_row: + print("### LightroomDB: {} / {}".format(tuple(lrdb_row), lrdb_row.keys())) + + # create a duplicate copy for later checking if something changed + data_set_original = data_set.copy() + # check if LR exists and use this to compare to XMP data + # is LR GPS and no XMP GPS => use LR and set XMP + # same for location names + # if missing in XMP but in LR -> set in XMP + # if missing in both do lookup in Maps if use_lightroom and lightroom_data_ok: - for key in data_set: - # if not the same (to original data) and passes overwrite check - if data_set[key] != data_set_original[key] and checkOverwrite(data_set_original[key], key, args.field_controls): - xmp.set_property(xmp_fields[key], key, data_set[key]) - write_file = True - if write_file: - count['lightroom'] += 1 - # if we have the write flag set, write data - if write_file: - if not args.test: - # use copyfile to create a backup copy - if not args.no_xmp_backup: - # check if there is another file with .BK. already there, if yes, get the max number and +1 it, if not set to 1 - bk_file_counter = getBackupFileCounter(xmp_file) - # copy to new backup file - copyfile(xmp_file, "{}.BK.{}{}".format(os.path.splitext(xmp_file)[0], bk_file_counter, os.path.splitext(xmp_file)[1])) - # write back to riginal file - with open(xmp_file, 'w') as fptr: - fptr.write(xmp.serialize_to_str(omit_packet_wrapper=True)) + # check lat/long separate + if lrdb_row['gpsLatitude'] and not data_set['GPSLatitude']: + # we need to convert to the Degree,Min.sec[NSEW] format + data_set['GPSLatitude'] = convertLatToDMS(lrdb_row['gpsLatitude']) + if lrdb_row['gpsLongitude'] and not data_set['GPSLongitude']: + data_set['GPSLongitude'] = convertLongToDMS(lrdb_row['gpsLongitude']) + # now check Location, City, etc + for loc in data_set_loc: + # overwrite original set (read from XMP) with LR data if original data is missing + if lrdb_row[loc] and not data_set[loc]: + data_set[loc] = lrdb_row[loc] + if args.debug: + print(f"### -> LR: {loc} => {lrdb_row[loc]}") + # base set done, now check if there is anything unset in the data_set, + # if yes do a lookup in maps + # run this through the overwrite checker to get unset if we have a forced overwrite + has_unset = False + failed = False + from_cache = False + for loc in data_set_loc: + if checkOverwrite(data_set[loc], loc, args.field_controls): + has_unset = True + if has_unset: + # check if lat/long is in cache + cache_key = f"{data_set['GPSLongitude']}#{data_set['GPSLatitude']}" + if args.debug: + print( + f"### *** CACHE: {cache_key}: " + f"{'NO' if cache_key not in data_cache else 'YES'}" + ) + # main chache check = identical + # second cache level check is on distance: + # default distance is 10m, can be set via flag + # check distance to previous cache entries (reverse newest to oldest) + # and match before we do google lookup + if cache_key not in data_cache: + has_fuzzy_cache = False + if args.fuzzy_distance: + shortest_distance = args.fuzzy_distance + best_match_latlong = '' + # check if we have fuzzy distance, if no valid found do maps lookup + for _cache_key in data_cache: + # split up cache key so we can use in the distance calc method + to_lat_long = _cache_key.split('#') + # get the distance based on current set + cached set + # print( + # f"Lookup f-long {data_set['GPSLongitude']} " + # f"f-lat {data_set['GPSLatitude']} " + # f"t-long {to_lat_long[0]} t-lat {to_lat_long[1]}" + # ) + distance = getDistance( + from_longitude=data_set['GPSLongitude'], + from_latitude=data_set['GPSLatitude'], + to_longitude=to_lat_long[0], + to_latitude=to_lat_long[1] + ) + if args.debug: + print( + f"### **= FUZZY CACHE: => distance: {distance} (m), " + f"shortest: {shortest_distance}" + ) + if distance <= shortest_distance: + # set new distance and keep current best matching location + shortest_distance = distance + best_match_latlong = _cache_key + has_fuzzy_cache = True + if args.debug: + print( + "### ***= FUZZY CACHE: YES => " + f"Best match: {best_match_latlong}" + ) + if not has_fuzzy_cache: + # get location from maps (google or openstreetmap) + maps_location = reverseGeolocate( + latitude=data_set['GPSLatitude'], + longitude=data_set['GPSLongitude'], + map_type=map_type + ) + # cache data with Lat/Long + data_cache[cache_key] = maps_location + from_cache = False + else: + maps_location = data_cache[best_match_latlong] + # cache this one, because the next one will match this one too + # we don't need to loop search again for the same fuzzy location + data_cache[cache_key] = maps_location + count['cache'] += 1 + count['fuzzy_cache'] += 1 + from_cache = True + else: + # load location from cache + maps_location = data_cache[cache_key] + count['cache'] += 1 + from_cache = True + # overwrite sets (note options check here) + if args.debug: + print(f"### Map Location ({map_type}): {maps_location}") + # must have at least the country set to write anything back + if maps_location['Country']: + for loc in data_set_loc: + # only write to XMP if overwrite check passes + if checkOverwrite(data_set_original[loc], loc, args.field_controls): + data_set[loc] = maps_location[loc] + xmp.set_property(xmp_fields[loc], loc, maps_location[loc]) + write_file = True + if write_file: + count['map'] += 1 + else: + print("(!) Could not geo loaction data ", end='') + failed = True else: - print("[TEST] Would write {} {}".format(data_set, xmp_file), end='') - if from_cache: - print("[UPDATED FROM CACHE]") + if args.debug: + print(f"Lightroom data use: {use_lightroom}, Lightroom data ok: {lightroom_data_ok}") + # check if the data_set differs from the original (LR db load) + # if yes write, else skip + if use_lightroom and lightroom_data_ok: + for key in data_set: + # if not the same (to original data) and passes overwrite check + if data_set[key] != data_set_original[key] and checkOverwrite(data_set_original[key], key, args.field_controls): + xmp.set_property(xmp_fields[key], key, data_set[key]) + write_file = True + if write_file: + count['lightroom'] += 1 + # if we have the write flag set, write data + if write_file: + if not args.test: + # use copyfile to create a backup copy + if not args.no_xmp_backup: + # check if there is another file with .BK. already there, if yes, get the max number and +1 it, if not set to 1 + bk_file_counter = getBackupFileCounter(xmp_file) + # copy to new backup file + copyfile(xmp_file, "{}.BK.{}{}".format(os.path.splitext(xmp_file)[0], bk_file_counter, os.path.splitext(xmp_file)[1])) + # write back to riginal file + with open(xmp_file, 'w') as fptr: + fptr.write(xmp.serialize_to_str(omit_packet_wrapper=True)) + else: + print("[TEST] Would write {} {}".format(data_set, xmp_file), end='') + if from_cache: + print("[UPDATED FROM CACHE]") + else: + print("[UPDATED]") + count['changed'] += 1 + elif failed: + print("[FAILED]") + count['failed'] += 1 + # log data to array for post print + failed_files.append(xmp_file) else: - print("[UPDATED]") - count['changed'] += 1 - elif failed: - print("[FAILED]") - count['failed'] += 1 - # log data to array for post print - failed_files.append(xmp_file) - else: - print("[SKIP]") - count['skipped'] += 1 + print("[SKIP]") + count['skipped'] += 1 -# close DB connection -if use_lightroom: - lrdb.close() - -# end stats only if we write -print("{}".format('=' * 40)) -print("XMP Files found : {:9,}".format(count['all'])) -if args.read_only: - print("XMP Files listed : {:9,}".format(count['listed'])) -if not args.read_only: - print("Updated : {:9,}".format(count['changed'])) - print("Skipped : {:9,}".format(count['skipped'])) - print("New GeoLocation from Map : {:9,}".format(count['map'])) - print("GeoLocation from Cache : {:9,}".format(count['cache'])) - print("GeoLocation from Fuzzy Cache : {:9,}".format(count['fuzzy_cache'])) - print("Failed reverse GeoLocate : {:9,}".format(count['failed'])) + # close DB connection if use_lightroom: - print("GeoLocaction from Lightroom : {:9,}".format(count['lightroom'])) - print("No Lightroom data found : {:9,}".format(count['not_found'])) - print("More than one found in LR : {:9,}".format(count['many_found'])) - # if we have failed data - if len(failed_files) > 0: - print("{}".format('-' * 40)) - print("Files that failed to update:") - print("{}".format(', '.join(failed_files))) + lrdb.close() + + # end stats only if we write + print(f"{'=' * 40}") + print(f"XMP Files found : {count['all']:9,}") + if args.read_only: + print(f"XMP Files listed : {count['listed']:9,}") + if not args.read_only: + print(f"Updated : {count['changed']:9,}") + print(f"Skipped : {count['skipped']:9,}") + print(f"New GeoLocation from Map : {count['map']:9,}") + print(f"GeoLocation from Cache : {count['cache']:9,}") + print(f"GeoLocation from Fuzzy Cache : {count['fuzzy_cache']:9,}") + print(f"Failed reverse GeoLocate : {count['failed']:9,}") + if use_lightroom: + print(f"GeoLocaction from Lightroom : {count['lightroom']:9,}") + print(f"No Lightroom data found : {count['not_found']:9,}") + print(f"More than one found in LR : {count['many_found']:9,}") + # if we have failed data + if len(failed_files) > 0: + print(f"{'-' * 40}") + print("Files that failed to update:") + print(f"{', '.join(failed_files)}") + + +############################################################## +# MAIN RUN +############################################################## + +main() # __END__ diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..4a9cc7b --- /dev/null +++ b/requirement.txt @@ -0,0 +1,8 @@ +certifi==2022.9.24 +charset-normalizer==2.1.1 +idna==3.4 +install==1.3.5 +python-xmp-toolkit==2.0.1 +pytz==2022.6 +requests==2.28.1 +urllib3==1.26.12