ReverseGeoLocate/reverse-geolocate
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

Reverse Geolocate Clean up run

Browse Source
This commit is contained in:
Clemens Schwaighofer
2022-11-11 23:08:41 +09:00
parent 65e352e8a4
commit b7de6d8119
4 changed files with 785 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{
"python.analysis.typeCheckingMode": "basic"
}
README.md
+16 -4
View File
@@ -4,16 +4,28 @@ Reverse GeoLocate from XMP sidecar files with optional LightRoom DB read
This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database. This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database.
#### Installing and setting up ## Development Setup
```sh
python3 -m venv .venv
source .venv/bin/activate
.venv/bin/python -m pip install -U pip setuptools wheel
```
Then install the requests and python-xmp-toolkit modules from below
## Installing and setting up
The script uses the following external non defauly python libraries The script uses the following external non defauly python libraries
* xmp toolkit * xmp toolkit
* requests * requests
install both with the pip3 command install both with the pip3 command
```
pip3 install requests ```sh
pip3 install python-xmp-toolkit pip install requests
pip install python-xmp-toolkit
``` ```
XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly. XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly.
bin/reverse_geolocate.py
+268 -216
View File
@@ -3,15 +3,14 @@
# AUTHOR : Clemens Schwaighofer # AUTHOR : Clemens Schwaighofer
# DATE : 2018/2/20 # DATE : 2018/2/20
# LICENSE: GPLv3 # LICENSE: GPLv3
# DESC : Set the reverse Geo location (name) from Lat/Long data in XMP files in a lightroom catalogue # DESC :
# Set the reverse Geo location (name) from Lat/Long data in XMP files
# in a lightroom catalogue
# * tries to get pre-set geo location from LR catalog # * tries to get pre-set geo location from LR catalog
# * if not found tries to get data from Google # * if not found tries to get data from Google
# * all data is translated into English with long vowl system (aka ou or oo is ō) # * all data is translated into English with long vowl system (aka ou or oo is ō)
# MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/) # MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/)
import argparse
import sqlite3
import requests
import configparser import configparser
import unicodedata import unicodedata
# import textwrap # import textwrap
@@ -19,10 +18,13 @@ import glob
import os import os
import sys import sys
import re import re
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta import argparse
from libxmp import XMPMeta, consts import sqlite3
from shutil import copyfile, get_terminal_size from shutil import copyfile, get_terminal_size
from math import ceil, radians, sin, cos, atan2, sqrt from math import ceil, radians, sin, cos, atan2, sqrt
import requests
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta
from libxmp import XMPMeta, consts
############################################################## ##############################################################
# FUNCTIONS # FUNCTIONS
@@ -32,13 +34,16 @@ from math import ceil, radians, sin, cos, atan2, sqrt
# this is used by isLatin and onlyLatinChars # this is used by isLatin and onlyLatinChars
cache_latin_letters = {} cache_latin_letters = {}
# ARGPARSE HELPERS # ARGPARSE HELPERS
# call: writable_dir_folder class WritableDirFolder(argparse.Action):
# checks if this is a writeable folder OR file """
# AND it works on nargs * checks if this is a writeable folder OR file
class writable_dir_folder(argparse.Action): AND it works on nargs *
Args:
argparse (_type_): _description_
"""
def __call__(self, parser, namespace, values, option_string=None): def __call__(self, parser, namespace, values, option_string=None):
# we loop through list (this is because of nargs *) # we loop through list (this is because of nargs *)
for prospective_dir in values: for prospective_dir in values:
@@ -632,42 +637,49 @@ def getBackupFileCounter(xmp_file):
return bk_file_counter return bk_file_counter
############################################################## ##############################################################
# ARGUMENT PARSNING # ARGUMENT PARSING
############################################################## ##############################################################
def argument_parser():
"""
Parses the command line arguments
parser = argparse.ArgumentParser( Returns:
Namespace: parsed arguments
"""
parser = argparse.ArgumentParser(
description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files', description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files',
# formatter_class=argparse.RawDescriptionHelpFormatter, # formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='Sample: (todo)' epilog='Sample: (todo)'
) )
# xmp folder (or folders), or file (or files) # xmp folder (or folders), or file (or files)
# note that the target directory or file needs to be writeable # note that the target directory or file needs to be writeable
parser.add_argument( parser.add_argument(
'-i', '-i',
'--include-source', '--include-source',
required=True, required=True,
nargs='*', nargs='*',
action=writable_dir_folder, action=WritableDirFolder,
dest='xmp_sources', dest='xmp_sources',
metavar='XMP SOURCE FOLDER', metavar='XMP SOURCE FOLDER',
help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here' help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here'
) )
# exclude folders # exclude folders
parser.add_argument( parser.add_argument(
'-x', '-x',
'--exclude-source', '--exclude-source',
nargs='*', nargs='*',
action=writable_dir_folder, action=WritableDirFolder,
dest='exclude_sources', dest='exclude_sources',
metavar='EXCLUDE XMP SOURCE FOLDER', metavar='EXCLUDE XMP SOURCE FOLDER',
help='Folders and files that will be excluded.' help='Folders and files that will be excluded.'
) )
# LR database (base folder) # LR database (base folder)
# get .lrcat file in this folder # get .lrcat file in this folder
parser.add_argument( parser.add_argument(
'-l', '-l',
'--lightroom', '--lightroom',
# required=True, # required=True,
@@ -675,22 +687,23 @@ parser.add_argument(
dest='lightroom_folder', dest='lightroom_folder',
metavar='LIGHTROOM FOLDER', metavar='LIGHTROOM FOLDER',
help='Lightroom catalogue base folder' help='Lightroom catalogue base folder'
) )
# strict LR check with base path next to the file base name # strict LR check with base path next to the file base name
parser.add_argument( parser.add_argument(
'-s', '-s',
'--strict', '--strict',
dest='lightroom_strict', dest='lightroom_strict',
action='store_true', action='store_true',
help='Do strict check for Lightroom files including Path in query' help='Do strict check for Lightroom files including Path in query'
) )
# set behaviour override # set behaviour override
# FLAG: default: only set not filled # FLAG: default: only set not filled
# other: overwrite all or overwrite if one is missing, overwrite specifc field (as defined below) # other: overwrite all or overwrite if one is missing,
# fields: Location, City, State, Country, CountryCode # overwrite specifc field (as defined below)
parser.add_argument( # fields: Location, City, State, Country, CountryCode
parser.add_argument(
'-f', '-f',
'--field', '--field',
action='append', action='append',
@@ -698,13 +711,17 @@ parser.add_argument(
choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'], choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'],
dest='field_controls', dest='field_controls',
metavar='<overwrite, location, city, state, country, countrycode>', metavar='<overwrite, location, city, state, country, countrycode>',
help='On default only set fields that are not set yet. Options are: '\ help=(
'Overwrite (write all new), Location, City, State, Country, CountryCode. '\ 'On default only set fields that are not set yet. Options are: '
'Multiple can be given for combination overwrite certain fields only or set only certain fields. '\ 'Overwrite (write all new), Location, City, State, Country, CountryCode. '
'If with overwrite the field will be overwritten if already set, else it will be always skipped.' 'Multiple can be given for combination overwrite certain fields only '
) 'or set only certain fields. '
'If with overwrite the field will be overwritten if already set, '
'else it will be always skipped.'
)
)
parser.add_argument( parser.add_argument(
'-d', '-d',
'--fuzzy-cache', '--fuzzy-cache',
type=str.lower, type=str.lower,
@@ -713,186 +730,190 @@ parser.add_argument(
const='10m', # default is 10m const='10m', # default is 10m
dest='fuzzy_distance', dest='fuzzy_distance',
metavar='FUZZY DISTANCE', metavar='FUZZY DISTANCE',
help='Allow fuzzy distance cache lookup. Optional distance can be given, '\ help=(
'if not set default of 10m is used. '\ 'Allow fuzzy distance cache lookup. Optional distance can be given, '
'if not set default of 10m is used. '
'Allowed argument is in the format of 12m or 12km' 'Allowed argument is in the format of 12m or 12km'
) )
)
# Google Maps API key to overcome restrictions # Google Maps API key to overcome restrictions
parser.add_argument( parser.add_argument(
'-g', '-g',
'--google', '--google',
dest='google_api_key', dest='google_api_key',
metavar='GOOGLE API KEY', metavar='GOOGLE API KEY',
help='Set a Google API Maps key to overcome the default lookup limitations' help='Set a Google API Maps key to overcome the default lookup limitations'
) )
# use open street maps # use open street maps
parser.add_argument( parser.add_argument(
'-o', '-o',
'--openstreetmap', '--openstreetmap',
dest='use_openstreetmap', dest='use_openstreetmap',
action='store_true', action='store_true',
help='Use openstreetmap instead of Google' help='Use openstreetmap instead of Google'
) )
# email of open street maps requests # email of open street maps requests
parser.add_argument( parser.add_argument(
'-e', '-e',
'--email', '--email',
dest='email', dest='email',
metavar='EMIL ADDRESS', metavar='EMIL ADDRESS',
help='An email address for OpenStreetMap' help='An email address for OpenStreetMap'
) )
# write api/email settings to config file # write api/email settings to config file
parser.add_argument( parser.add_argument(
'-w', '-w',
'--write-settings', '--write-settings',
dest='config_write', dest='config_write',
action='store_true', action='store_true',
help='Write Google API or OpenStreetMap email to config file' help='Write Google API or OpenStreetMap email to config file'
) )
# only read data and print on screen, do not write anything # only read data and print on screen, do not write anything
parser.add_argument( parser.add_argument(
'-r', '-r',
'--read-only', '--read-only',
dest='read_only', dest='read_only',
action='store_true', action='store_true',
help='Read current values from the XMP file only, do not read from LR or lookup any data and write back' help=(
) 'Read current values from the XMP file only, '
'do not read from LR or lookup any data and write back'
)
)
# only list unset ones # only list unset ones
parser.add_argument( parser.add_argument(
'-u', '-u',
'--unset-only', '--unset-only',
dest='unset_only', dest='unset_only',
action='store_true', action='store_true',
help='Only list unset XMP files' help='Only list unset XMP files'
) )
# only list unset GPS codes # only list unset GPS codes
parser.add_argument( parser.add_argument(
'-p', '-p',
'--unset-gps-only', '--unset-gps-only',
dest='unset_gps_only', dest='unset_gps_only',
action='store_true', action='store_true',
help='Only list unset XMP files for GPS fields' help='Only list unset XMP files for GPS fields'
) )
# don't try to do auto adjust in list view # don't try to do auto adjust in list view
parser.add_argument( parser.add_argument(
'-a', '-a',
'--no-autoadjust', '--no-autoadjust',
dest='no_autoadjust', dest='no_autoadjust',
action='store_true', action='store_true',
help='Don\'t try to auto adjust columns' help='Don\'t try to auto adjust columns'
) )
# compact view, compresses columns down to a minimum # compact view, compresses columns down to a minimum
parser.add_argument( parser.add_argument(
'-c', '-c',
'--compact', '--compact',
dest='compact_view', dest='compact_view',
action='store_true', action='store_true',
help='Very compact list view' help='Very compact list view'
) )
# Do not create backup files # Do not create backup files
parser.add_argument( parser.add_argument(
'-n', '-n',
'--nobackup', '--nobackup',
dest='no_xmp_backup', dest='no_xmp_backup',
action='store_true', action='store_true',
help='Do not create a backup from the XMP file' help='Do not create a backup from the XMP file'
) )
# verbose args for more detailed output # verbose args for more detailed output
parser.add_argument( parser.add_argument(
'-v', '-v',
'--verbose', '--verbose',
action='count', action='count',
dest='verbose', dest='verbose',
help='Set verbose output level' help='Set verbose output level'
) )
# debug flag # debug flag
parser.add_argument('--debug', action='store_true', dest='debug', help='Set detailed debug output') parser.add_argument(
# test flag '--debug', action='store_true', dest='debug', help='Set detailed debug output'
parser.add_argument('--test', action='store_true', dest='test', help='Do not write data back to file') )
# test flag
parser.add_argument(
'--test', action='store_true', dest='test', help='Do not write data back to file'
)
# read in the argumens # read in the argumens
args = parser.parse_args() return parser.parse_args()
############################################################## ##############################################################
# MAIN CODE # MAIN CODE
############################################################## ##############################################################
# init verbose to 0 if not set def main():
if not args.verbose: """
Main code run
"""
args = argument_parser()
# init verbose to 0 if not set
if not args.verbose:
args.verbose = 0 args.verbose = 0
# init exclude source to list if not set # init exclude source to list if not set
if not args.exclude_sources: if not args.exclude_sources:
args.exclude_sources = [] args.exclude_sources = []
# init args unset (for list view) with 0 if unset # init args unset (for list view) with 0 if unset
if not args.unset_only: if not args.unset_only:
args.unset_only = 0 args.unset_only = 0
if args.debug: if args.debug:
print("### ARGUMENT VARS: I: {incl}, X: {excl}, L: {lr}, F: {fc}, D: {fdist}, M: {osm}, G: {gp}, E: {em}, R: {read}, U: {us}, A: {adj}, C: {cmp}, N: {nbk}, W: {wrc}, V: {v}, D: {d}, T: {t}".format( print(
incl=args.xmp_sources, "### ARGUMENT VARS: "
excl=args.exclude_sources, f"I: {args.xmp_sources}, X: {args.exclude_sources}, L: {args.lightroom_folder}, "
lr=args.lightroom_folder, f"F: {args.field_controls}, D: {args.fuzzy_distance}, M: {args.use_openstreetmap}, "
fc=args.field_controls, f"G: {args.google_api_key}, E: {args.email}, R: {args.read_only}, U: {args.unset_only}, "
fdist=args.fuzzy_distance, f"A: {args.no_autoadjust}, C: {args.compact_view}, N: {args.no_xmp_backup}, "
osm=args.use_openstreetmap, f"W: {args.config_write}, V: {args.verbose}, D: {args.debug}, T: {args.test}"
gp=args.google_api_key, )
em=args.email,
read=args.read_only,
us=args.unset_only,
adj=args.no_autoadjust,
cmp=args.compact_view,
nbk=args.no_xmp_backup,
wrc=args.config_write,
v=args.verbose,
d=args.debug,
t=args.test
))
# error flag # error flag
error = False error = False
# set search map type # set search map type
map_type = 'google' if not args.use_openstreetmap else 'openstreetmap' map_type = 'google' if not args.use_openstreetmap else 'openstreetmap'
# if -g and -o, error # if -g and -o, error
if args.google_api_key and args.use_openstreetmap: if args.google_api_key and args.use_openstreetmap:
print("You cannot set a Google API key and use OpenStreetMap at the same time") print("You cannot set a Google API key and use OpenStreetMap at the same time")
error = True error = True
# or if -g and -e # or if -g and -e
if args.google_api_key and args.email: if args.google_api_key and args.email:
print("You cannot set a Google API key and OpenStreetMap email at the same time") print("You cannot set a Google API key and OpenStreetMap email at the same time")
error = True error = True
# or -e and no -o # or -e and no -o
if args.email and not args.use_openstreetmap: if args.email and not args.use_openstreetmap:
print("You cannot set an OpenStreetMap email and not use OpenStreetMap") print("You cannot set an OpenStreetMap email and not use OpenStreetMap")
error = True error = True
# if email and not basic valid email (@ .) # if email and not basic valid email (@ .)
if args.email: if args.email:
if not re.match(r'^.+@.+\.[A-Za-z]{1,}$', args.email): if not re.match(r'^.+@.+\.[A-Za-z]{1,}$', args.email):
print("Not a valid email for OpenStreetMap: {}".format(args.email)) print("Not a valid email for OpenStreetMap: {}".format(args.email))
error = True error = True
# on error exit here # on error exit here
if error: if error:
sys.exit(1) sys.exit(1)
config = configparser.ConfigParser() config = configparser.ConfigParser()
# try to find config file in following order # try to find config file in following order
# $HOME/.config/ # $HOME/.config/
config_file = 'reverse_geolocate.cfg' config_file = 'reverse_geolocate.cfg'
config_folder = os.path.expanduser('~/.config/reverseGeolocate/') config_folder = os.path.expanduser('~/.config/reverseGeolocate/')
config_data = '{}{}'.format(config_folder, config_file) config_data = '{}{}'.format(config_folder, config_file)
# if file exists read, if not skip unless we have write flag and google api or openstreetmaps email # if file exists read, if not skip unless we have write flag and google api or openstreetmaps email
if os.path.isfile(config_data): if os.path.isfile(config_data):
config.read(config_data) config.read(config_data)
# check if api group & setting is there. also never overwrite argument given data # check if api group & setting is there. also never overwrite argument given data
if 'API' in config: if 'API' in config:
@@ -902,8 +923,8 @@ if os.path.isfile(config_data):
if 'openstreetmapemail' in config['API']: if 'openstreetmapemail' in config['API']:
if not args.email: if not args.email:
args.email = config['API']['openstreetmapemail'] args.email = config['API']['openstreetmapemail']
# write data if exists and changed # write data if exists and changed
if args.config_write and (args.google_api_key or args.email): if args.config_write and (args.google_api_key or args.email):
config_change = False config_change = False
# check if new value differs, if yes, change and write # check if new value differs, if yes, change and write
if 'API' not in config: if 'API' not in config:
@@ -920,21 +941,21 @@ if args.config_write and (args.google_api_key or args.email):
os.makedirs(config_folder) os.makedirs(config_folder)
with open(config_data, 'w') as fptr: with open(config_data, 'w') as fptr:
config.write(fptr) config.write(fptr)
if args.debug: if args.debug:
print("### OVERRIDE API: G: {}, O: {}".format(args.google_api_key, args.email)) print("### OVERRIDE API: G: {}, O: {}".format(args.google_api_key, args.email))
# The XMP fields const lookup values # The XMP fields const lookup values
# XML/XMP # XML/XMP
# READ: # READ:
# exif:GPSLatitude # exif:GPSLatitude
# exif:GPSLongitude # exif:GPSLongitude
# READ for if filled # READ for if filled
# Iptc4xmpCore:Location # Iptc4xmpCore:Location
# photoshop:City # photoshop:City
# photoshop:State # photoshop:State
# photoshop:Country # photoshop:Country
# Iptc4xmpCore:CountryCode # Iptc4xmpCore:CountryCode
xmp_fields = { xmp_fields = {
'GPSLatitude': consts.XMP_NS_EXIF, # EXIF GPSLat/Long are stored in Degree,Min.Sec[NESW] format 'GPSLatitude': consts.XMP_NS_EXIF, # EXIF GPSLat/Long are stored in Degree,Min.Sec[NESW] format
'GPSLongitude': consts.XMP_NS_EXIF, 'GPSLongitude': consts.XMP_NS_EXIF,
'Location': consts.XMP_NS_IPTCCore, 'Location': consts.XMP_NS_IPTCCore,
@@ -942,11 +963,11 @@ xmp_fields = {
'State': consts.XMP_NS_Photoshop, 'State': consts.XMP_NS_Photoshop,
'Country': consts.XMP_NS_Photoshop, 'Country': consts.XMP_NS_Photoshop,
'CountryCode': consts.XMP_NS_IPTCCore 'CountryCode': consts.XMP_NS_IPTCCore
} }
# non lat/long fields (for loc loops) # non lat/long fields (for loc loops)
data_set_loc = ('Location', 'City', 'State', 'Country', 'CountryCode') data_set_loc = ('Location', 'City', 'State', 'Country', 'CountryCode')
# one xmp data set # one xmp data set
data_set = { data_set = {
'GPSLatitude': '', 'GPSLatitude': '',
'GPSLongitude': '', 'GPSLongitude': '',
'Location': '', 'Location': '',
@@ -954,22 +975,22 @@ data_set = {
'State': '', 'State': '',
'Country': '', 'Country': '',
'CountryCode': '' 'CountryCode': ''
} }
# original set for compare (is constant unchanged) # original set for compare (is constant unchanged)
data_set_original = {} data_set_original = {}
# cache set to avoid double lookups for identical Lat/Ling # cache set to avoid double lookups for identical Lat/Ling
data_cache = {} data_cache = {}
# work files, all files + folders we need to work on # work files, all files + folders we need to work on
work_files = [] work_files = []
# all failed files # all failed files
failed_files = [] failed_files = []
# use lightroom # use lightroom
use_lightroom = False use_lightroom = False
# cursors & query # cursors & query
query = '' query = ''
cur = '' cur = ''
# count variables # count variables
count = { count = {
'all': 0, 'all': 0,
'listed': 0, 'listed': 0,
'read': 0, 'read': 0,
@@ -982,10 +1003,10 @@ count = {
'skipped': 0, 'skipped': 0,
'not_found': 0, 'not_found': 0,
'many_found': 0, 'many_found': 0,
} }
# do lightroom stuff only if we have the lightroom folder # do lightroom stuff only if we have the lightroom folder
if args.lightroom_folder: if args.lightroom_folder:
# query string for lightroom DB check # query string for lightroom DB check
query = 'SELECT Adobe_images.id_local, AgLibraryFile.baseName, AgLibraryRootFolder.absolutePath, AgLibraryRootFolder.name as realtivePath, AgLibraryFolder.pathFromRoot, AgLibraryFile.originalFilename, ' query = 'SELECT Adobe_images.id_local, AgLibraryFile.baseName, AgLibraryRootFolder.absolutePath, AgLibraryRootFolder.name as realtivePath, AgLibraryFolder.pathFromRoot, AgLibraryFile.originalFilename, '
query += 'AgHarvestedExifMetadata.gpsLatitude, AgHarvestedExifMetadata.gpsLongitude, ' query += 'AgHarvestedExifMetadata.gpsLatitude, AgHarvestedExifMetadata.gpsLongitude, '
@@ -1024,15 +1045,15 @@ if args.lightroom_folder:
if args.debug: if args.debug:
print("### USE Lightroom {}".format(use_lightroom)) print("### USE Lightroom {}".format(use_lightroom))
# on error exit here # on error exit here
if error: if error:
sys.exit(1) sys.exit(1)
# init the XML meta for handling # init the XML meta for handling
xmp = XMPMeta() xmp = XMPMeta()
# loop through the xmp_sources (folder or files) and read in the XMP data for LAT/LONG, other data # loop through the xmp_sources (folder or files) and read in the XMP data for LAT/LONG, other data
for xmp_file_source in args.xmp_sources: for xmp_file_source in args.xmp_sources:
# if folder, open and loop # if folder, open and loop
# NOTE: we do check for folders in there, if there are we recourse traverse them # NOTE: we do check for folders in there, if there are we recourse traverse them
# also check that folder is not in exclude list # also check that folder is not in exclude list
@@ -1056,11 +1077,11 @@ for xmp_file_source in args.xmp_sources:
if xmp_file_source not in work_files and xmp_file_source not in args.exclude_sources: if xmp_file_source not in work_files and xmp_file_source not in args.exclude_sources:
work_files.append(xmp_file_source) work_files.append(xmp_file_source)
count['all'] += 1 count['all'] += 1
if args.debug: if args.debug:
print("### Work Files {}".format(work_files)) print("### Work Files {}".format(work_files))
# if we have read only we print list format style # if we have read only we print list format style
if args.read_only: if args.read_only:
# adjust the output width for the list view # adjust the output width for the list view
format_length = outputListWidthAdjust() format_length = outputListWidthAdjust()
@@ -1089,8 +1110,8 @@ if args.read_only:
# header title # header title
# seperator line # seperator line
header_line = '''{} header_line = '''{}
{} {}
{}'''.format( {}'''.format(
'> Page {page_no:,}/{page_all:,}', # can later be set to something else, eg page numbers '> Page {page_no:,}/{page_all:,}', # can later be set to something else, eg page numbers
# pre replace path length before we add the header titles # pre replace path length before we add the header titles
format_line.format( format_line.format(
@@ -1129,9 +1150,9 @@ if args.read_only:
if not work_files: if not work_files:
print("{:<60}".format('[!!!] No files found')) print("{:<60}".format('[!!!] No files found'))
# ### MAIN WORK LOOP # ### MAIN WORK LOOP
# now we just loop through each file and work on them # now we just loop through each file and work on them
for xmp_file in work_files: # noqa: C901 for xmp_file in work_files: # noqa: C901
if not args.read_only: if not args.read_only:
print("---> {}: ".format(xmp_file), end='') print("---> {}: ".format(xmp_file), end='')
@@ -1230,8 +1251,9 @@ for xmp_file in work_files: # noqa: C901
if lrdb_row[loc] and not data_set[loc]: if lrdb_row[loc] and not data_set[loc]:
data_set[loc] = lrdb_row[loc] data_set[loc] = lrdb_row[loc]
if args.debug: if args.debug:
print("### -> LR: {} => {}".format(loc, lrdb_row[loc])) print(f"### -> LR: {loc} => {lrdb_row[loc]}")
# base set done, now check if there is anything unset in the data_set, if yes do a lookup in maps # base set done, now check if there is anything unset in the data_set,
# if yes do a lookup in maps
# run this through the overwrite checker to get unset if we have a forced overwrite # run this through the overwrite checker to get unset if we have a forced overwrite
has_unset = False has_unset = False
failed = False failed = False
@@ -1241,13 +1263,17 @@ for xmp_file in work_files: # noqa: C901
has_unset = True has_unset = True
if has_unset: if has_unset:
# check if lat/long is in cache # check if lat/long is in cache
cache_key = '{}#{}'.format(data_set['GPSLongitude'], data_set['GPSLatitude']) cache_key = f"{data_set['GPSLongitude']}#{data_set['GPSLatitude']}"
if args.debug: if args.debug:
print("### *** CACHE: {}: {}".format(cache_key, 'NO' if cache_key not in data_cache else 'YES')) print(
f"### *** CACHE: {cache_key}: "
f"{'NO' if cache_key not in data_cache else 'YES'}"
)
# main chache check = identical # main chache check = identical
# second cache level check is on distance: # second cache level check is on distance:
# default distance is 10m, can be set via flag # default distance is 10m, can be set via flag
# check distance to previous cache entries (reverse newest to oldest) and match before we do google lookup # check distance to previous cache entries (reverse newest to oldest)
# and match before we do google lookup
if cache_key not in data_cache: if cache_key not in data_cache:
has_fuzzy_cache = False has_fuzzy_cache = False
if args.fuzzy_distance: if args.fuzzy_distance:
@@ -1258,20 +1284,39 @@ for xmp_file in work_files: # noqa: C901
# split up cache key so we can use in the distance calc method # split up cache key so we can use in the distance calc method
to_lat_long = _cache_key.split('#') to_lat_long = _cache_key.split('#')
# get the distance based on current set + cached set # get the distance based on current set + cached set
# print("Lookup f-long {} f-lat {} t-long {} t-lat {}".format(data_set['GPSLongitude'], data_set['GPSLatitude'], to_lat_long[0], to_lat_long[1])) # print(
distance = getDistance(from_longitude=data_set['GPSLongitude'], from_latitude=data_set['GPSLatitude'], to_longitude=to_lat_long[0], to_latitude=to_lat_long[1]) # f"Lookup f-long {data_set['GPSLongitude']} "
# f"f-lat {data_set['GPSLatitude']} "
# f"t-long {to_lat_long[0]} t-lat {to_lat_long[1]}"
# )
distance = getDistance(
from_longitude=data_set['GPSLongitude'],
from_latitude=data_set['GPSLatitude'],
to_longitude=to_lat_long[0],
to_latitude=to_lat_long[1]
)
if args.debug: if args.debug:
print("### **= FUZZY CACHE: => distance: {} (m), shortest: {}".format(distance, shortest_distance)) print(
f"### **= FUZZY CACHE: => distance: {distance} (m), "
f"shortest: {shortest_distance}"
)
if distance <= shortest_distance: if distance <= shortest_distance:
# set new distance and keep current best matching location # set new distance and keep current best matching location
shortest_distance = distance shortest_distance = distance
best_match_latlong = _cache_key best_match_latlong = _cache_key
has_fuzzy_cache = True has_fuzzy_cache = True
if args.debug: if args.debug:
print("### ***= FUZZY CACHE: YES => Best match: {}".format(best_match_latlong)) print(
"### ***= FUZZY CACHE: YES => "
f"Best match: {best_match_latlong}"
)
if not has_fuzzy_cache: if not has_fuzzy_cache:
# get location from maps (google or openstreetmap) # get location from maps (google or openstreetmap)
maps_location = reverseGeolocate(latitude=data_set['GPSLatitude'], longitude=data_set['GPSLongitude'], map_type=map_type) maps_location = reverseGeolocate(
latitude=data_set['GPSLatitude'],
longitude=data_set['GPSLongitude'],
map_type=map_type
)
# cache data with Lat/Long # cache data with Lat/Long
data_cache[cache_key] = maps_location data_cache[cache_key] = maps_location
from_cache = False from_cache = False
@@ -1290,7 +1335,7 @@ for xmp_file in work_files: # noqa: C901
from_cache = True from_cache = True
# overwrite sets (note options check here) # overwrite sets (note options check here)
if args.debug: if args.debug:
print("### Map Location ({}): {}".format(map_type, maps_location)) print(f"### Map Location ({map_type}): {maps_location}")
# must have at least the country set to write anything back # must have at least the country set to write anything back
if maps_location['Country']: if maps_location['Country']:
for loc in data_set_loc: for loc in data_set_loc:
@@ -1306,7 +1351,7 @@ for xmp_file in work_files: # noqa: C901
failed = True failed = True
else: else:
if args.debug: if args.debug:
print("Lightroom data use: {}, Lightroom data ok: {}".format(use_lightroom, lightroom_data_ok)) print(f"Lightroom data use: {use_lightroom}, Lightroom data ok: {lightroom_data_ok}")
# check if the data_set differs from the original (LR db load) # check if the data_set differs from the original (LR db load)
# if yes write, else skip # if yes write, else skip
if use_lightroom and lightroom_data_ok: if use_lightroom and lightroom_data_ok:
@@ -1345,30 +1390,37 @@ for xmp_file in work_files: # noqa: C901
print("[SKIP]") print("[SKIP]")
count['skipped'] += 1 count['skipped'] += 1
# close DB connection # close DB connection
if use_lightroom: if use_lightroom:
lrdb.close() lrdb.close()
# end stats only if we write # end stats only if we write
print("{}".format('=' * 40)) print(f"{'=' * 40}")
print("XMP Files found : {:9,}".format(count['all'])) print(f"XMP Files found : {count['all']:9,}")
if args.read_only: if args.read_only:
print("XMP Files listed : {:9,}".format(count['listed'])) print(f"XMP Files listed : {count['listed']:9,}")
if not args.read_only: if not args.read_only:
print("Updated : {:9,}".format(count['changed'])) print(f"Updated : {count['changed']:9,}")
print("Skipped : {:9,}".format(count['skipped'])) print(f"Skipped : {count['skipped']:9,}")
print("New GeoLocation from Map : {:9,}".format(count['map'])) print(f"New GeoLocation from Map : {count['map']:9,}")
print("GeoLocation from Cache : {:9,}".format(count['cache'])) print(f"GeoLocation from Cache : {count['cache']:9,}")
print("GeoLocation from Fuzzy Cache : {:9,}".format(count['fuzzy_cache'])) print(f"GeoLocation from Fuzzy Cache : {count['fuzzy_cache']:9,}")
print("Failed reverse GeoLocate : {:9,}".format(count['failed'])) print(f"Failed reverse GeoLocate : {count['failed']:9,}")
if use_lightroom: if use_lightroom:
print("GeoLocaction from Lightroom : {:9,}".format(count['lightroom'])) print(f"GeoLocaction from Lightroom : {count['lightroom']:9,}")
print("No Lightroom data found : {:9,}".format(count['not_found'])) print(f"No Lightroom data found : {count['not_found']:9,}")
print("More than one found in LR : {:9,}".format(count['many_found'])) print(f"More than one found in LR : {count['many_found']:9,}")
# if we have failed data # if we have failed data
if len(failed_files) > 0: if len(failed_files) > 0:
print("{}".format('-' * 40)) print(f"{'-' * 40}")
print("Files that failed to update:") print("Files that failed to update:")
print("{}".format(', '.join(failed_files))) print(f"{', '.join(failed_files)}")
##############################################################
# MAIN RUN
##############################################################
main()
# __END__ # __END__
requirement.txt
+8
View File
@@ -0,0 +1,8 @@
certifi==2022.9.24
charset-normalizer==2.1.1
idna==3.4
install==1.3.5
python-xmp-toolkit==2.0.1
pytz==2022.6
requests==2.28.1
urllib3==1.26.12