ReverseGeoLocate/reverse-geolocate
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

Reverse Geolocate Clean up run

Browse Source
This commit is contained in:
Clemens Schwaighofer
2022-11-11 23:08:41 +09:00
parent 65e352e8a4
commit b7de6d8119
4 changed files with 785 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"python.analysis.typeCheckingMode": "basic"
}

20
README.md
View File

@@ -4,16 +4,28 @@ Reverse GeoLocate from XMP sidecar files with optional LightRoom DB read
This script will update any of the Country Code, Country, State, City and Location data that is missing in sidecard files. If a Lightroom DB is set, it will read any data from the database and fill in the fields before it tries to get the location name from google with the Latitude and Longitude found in either the XMP sidecar file or the LR database.
#### Installing and setting up
## Development Setup
```sh
python3 -m venv .venv
source .venv/bin/activate
.venv/bin/python -m pip install -U pip setuptools wheel
```
Then install the requests and python-xmp-toolkit modules from below
## Installing and setting up
The script uses the following external non defauly python libraries
* xmp toolkit
* requests
install both with the pip3 command
```
pip3 install requests
pip3 install python-xmp-toolkit
```sh
pip install requests
pip install python-xmp-toolkit
```
XMP Toolkit also needs the [Exempi Library](http://libopenraw.freedesktop.org/wiki/Exempi). This one can be install via brew or macports directly.

192
bin/reverse_geolocate.py
View File

@@ -3,15 +3,14 @@
# AUTHOR : Clemens Schwaighofer
# DATE : 2018/2/20
# LICENSE: GPLv3
# DESC : Set the reverse Geo location (name) from Lat/Long data in XMP files in a lightroom catalogue
# DESC :
# Set the reverse Geo location (name) from Lat/Long data in XMP files
# in a lightroom catalogue
# * tries to get pre-set geo location from LR catalog
# * if not found tries to get data from Google
# * all data is translated into English with long vowl system (aka ou or oo is ō)
# MUST HAVE: Python XMP Toolkit (http://python-xmp-toolkit.readthedocs.io/)
import argparse
import sqlite3
import requests
import configparser
import unicodedata
# import textwrap
@@ -19,10 +18,13 @@ import glob
import os
import sys
import re
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta
from libxmp import XMPMeta, consts
import argparse
import sqlite3
from shutil import copyfile, get_terminal_size
from math import ceil, radians, sin, cos, atan2, sqrt
import requests
# Note XMPFiles does not work with sidecar files, need to read via XMPMeta
from libxmp import XMPMeta, consts
##############################################################
# FUNCTIONS
@@ -32,13 +34,16 @@ from math import ceil, radians, sin, cos, atan2, sqrt
# this is used by isLatin and onlyLatinChars
cache_latin_letters = {}
# ARGPARSE HELPERS
# call: writable_dir_folder
# checks if this is a writeable folder OR file
# AND it works on nargs *
class writable_dir_folder(argparse.Action):
class WritableDirFolder(argparse.Action):
"""
checks if this is a writeable folder OR file
AND it works on nargs *
Args:
argparse (_type_): _description_
"""
def __call__(self, parser, namespace, values, option_string=None):
# we loop through list (this is because of nargs *)
for prospective_dir in values:
@@ -632,9 +637,16 @@ def getBackupFileCounter(xmp_file):
return bk_file_counter
##############################################################
# ARGUMENT PARSNING
# ARGUMENT PARSING
##############################################################
def argument_parser():
"""
Parses the command line arguments
Returns:
Namespace: parsed arguments
"""
parser = argparse.ArgumentParser(
description='Reverse Geoencoding based on set Latitude/Longitude data in XMP files',
@@ -649,7 +661,7 @@ parser.add_argument(
'--include-source',
required=True,
nargs='*',
action=writable_dir_folder,
action=WritableDirFolder,
dest='xmp_sources',
metavar='XMP SOURCE FOLDER',
help='The source folder or folders with the XMP files that need reverse geo encoding to be set. Single XMP files can be given here'
@@ -659,7 +671,7 @@ parser.add_argument(
'-x',
'--exclude-source',
nargs='*',
action=writable_dir_folder,
action=WritableDirFolder,
dest='exclude_sources',
metavar='EXCLUDE XMP SOURCE FOLDER',
help='Folders and files that will be excluded.'
@@ -688,7 +700,8 @@ parser.add_argument(
# set behaviour override
# FLAG: default: only set not filled
# other: overwrite all or overwrite if one is missing, overwrite specifc field (as defined below)
# other: overwrite all or overwrite if one is missing,
# overwrite specifc field (as defined below)
# fields: Location, City, State, Country, CountryCode
parser.add_argument(
'-f',
@@ -698,10 +711,14 @@ parser.add_argument(
choices=['overwrite', 'location', 'city', 'state', 'country', 'countrycode'],
dest='field_controls',
metavar='<overwrite, location, city, state, country, countrycode>',
help='On default only set fields that are not set yet. Options are: '\
'Overwrite (write all new), Location, City, State, Country, CountryCode. '\
'Multiple can be given for combination overwrite certain fields only or set only certain fields. '\
'If with overwrite the field will be overwritten if already set, else it will be always skipped.'
help=(
'On default only set fields that are not set yet. Options are: '
'Overwrite (write all new), Location, City, State, Country, CountryCode. '
'Multiple can be given for combination overwrite certain fields only '
'or set only certain fields. '
'If with overwrite the field will be overwritten if already set, '
'else it will be always skipped.'
)
)
parser.add_argument(
@@ -713,10 +730,12 @@ parser.add_argument(
const='10m', # default is 10m
dest='fuzzy_distance',
metavar='FUZZY DISTANCE',
help='Allow fuzzy distance cache lookup. Optional distance can be given, '\
'if not set default of 10m is used. '\
help=(
'Allow fuzzy distance cache lookup. Optional distance can be given, '
'if not set default of 10m is used. '
'Allowed argument is in the format of 12m or 12km'
)
)
# Google Maps API key to overcome restrictions
parser.add_argument(
@@ -760,7 +779,10 @@ parser.add_argument(
'--read-only',
dest='read_only',
action='store_true',
help='Read current values from the XMP file only, do not read from LR or lookup any data and write back'
help=(
'Read current values from the XMP file only, '
'do not read from LR or lookup any data and write back'
)
)
# only list unset ones
@@ -818,17 +840,27 @@ parser.add_argument(
)
# debug flag
parser.add_argument('--debug', action='store_true', dest='debug', help='Set detailed debug output')
parser.add_argument(
'--debug', action='store_true', dest='debug', help='Set detailed debug output'
)
# test flag
parser.add_argument('--test', action='store_true', dest='test', help='Do not write data back to file')
parser.add_argument(
'--test', action='store_true', dest='test', help='Do not write data back to file'
)
# read in the argumens
args = parser.parse_args()
return parser.parse_args()
##############################################################
# MAIN CODE
##############################################################
def main():
"""
Main code run
"""
args = argument_parser()
# init verbose to 0 if not set
if not args.verbose:
args.verbose = 0
@@ -840,25 +872,14 @@ if not args.unset_only:
args.unset_only = 0
if args.debug:
print("### ARGUMENT VARS: I: {incl}, X: {excl}, L: {lr}, F: {fc}, D: {fdist}, M: {osm}, G: {gp}, E: {em}, R: {read}, U: {us}, A: {adj}, C: {cmp}, N: {nbk}, W: {wrc}, V: {v}, D: {d}, T: {t}".format(
incl=args.xmp_sources,
excl=args.exclude_sources,
lr=args.lightroom_folder,
fc=args.field_controls,
fdist=args.fuzzy_distance,
osm=args.use_openstreetmap,
gp=args.google_api_key,
em=args.email,
read=args.read_only,
us=args.unset_only,
adj=args.no_autoadjust,
cmp=args.compact_view,
nbk=args.no_xmp_backup,
wrc=args.config_write,
v=args.verbose,
d=args.debug,
t=args.test
))
print(
"### ARGUMENT VARS: "
f"I: {args.xmp_sources}, X: {args.exclude_sources}, L: {args.lightroom_folder}, "
f"F: {args.field_controls}, D: {args.fuzzy_distance}, M: {args.use_openstreetmap}, "
f"G: {args.google_api_key}, E: {args.email}, R: {args.read_only}, U: {args.unset_only}, "
f"A: {args.no_autoadjust}, C: {args.compact_view}, N: {args.no_xmp_backup}, "
f"W: {args.config_write}, V: {args.verbose}, D: {args.debug}, T: {args.test}"
)
# error flag
error = False
@@ -1230,8 +1251,9 @@ for xmp_file in work_files: # noqa: C901
if lrdb_row[loc] and not data_set[loc]:
data_set[loc] = lrdb_row[loc]
if args.debug:
print("### -> LR: {} => {}".format(loc, lrdb_row[loc]))
# base set done, now check if there is anything unset in the data_set, if yes do a lookup in maps
print(f"### -> LR: {loc} => {lrdb_row[loc]}")
# base set done, now check if there is anything unset in the data_set,
# if yes do a lookup in maps
# run this through the overwrite checker to get unset if we have a forced overwrite
has_unset = False
failed = False
@@ -1241,13 +1263,17 @@ for xmp_file in work_files: # noqa: C901
has_unset = True
if has_unset:
# check if lat/long is in cache
cache_key = '{}#{}'.format(data_set['GPSLongitude'], data_set['GPSLatitude'])
cache_key = f"{data_set['GPSLongitude']}#{data_set['GPSLatitude']}"
if args.debug:
print("### *** CACHE: {}: {}".format(cache_key, 'NO' if cache_key not in data_cache else 'YES'))
print(
f"### *** CACHE: {cache_key}: "
f"{'NO' if cache_key not in data_cache else 'YES'}"
)
# main chache check = identical
# second cache level check is on distance:
# default distance is 10m, can be set via flag
# check distance to previous cache entries (reverse newest to oldest) and match before we do google lookup
# check distance to previous cache entries (reverse newest to oldest)
# and match before we do google lookup
if cache_key not in data_cache:
has_fuzzy_cache = False
if args.fuzzy_distance:
@@ -1258,20 +1284,39 @@ for xmp_file in work_files: # noqa: C901
# split up cache key so we can use in the distance calc method
to_lat_long = _cache_key.split('#')
# get the distance based on current set + cached set
# print("Lookup f-long {} f-lat {} t-long {} t-lat {}".format(data_set['GPSLongitude'], data_set['GPSLatitude'], to_lat_long[0], to_lat_long[1]))
distance = getDistance(from_longitude=data_set['GPSLongitude'], from_latitude=data_set['GPSLatitude'], to_longitude=to_lat_long[0], to_latitude=to_lat_long[1])
# print(
# f"Lookup f-long {data_set['GPSLongitude']} "
# f"f-lat {data_set['GPSLatitude']} "
# f"t-long {to_lat_long[0]} t-lat {to_lat_long[1]}"
# )
distance = getDistance(
from_longitude=data_set['GPSLongitude'],
from_latitude=data_set['GPSLatitude'],
to_longitude=to_lat_long[0],
to_latitude=to_lat_long[1]
)
if args.debug:
print("### **= FUZZY CACHE: => distance: {} (m), shortest: {}".format(distance, shortest_distance))
print(
f"### **= FUZZY CACHE: => distance: {distance} (m), "
f"shortest: {shortest_distance}"
)
if distance <= shortest_distance:
# set new distance and keep current best matching location
shortest_distance = distance
best_match_latlong = _cache_key
has_fuzzy_cache = True
if args.debug:
print("### ***= FUZZY CACHE: YES => Best match: {}".format(best_match_latlong))
print(
"### ***= FUZZY CACHE: YES => "
f"Best match: {best_match_latlong}"
)
if not has_fuzzy_cache:
# get location from maps (google or openstreetmap)
maps_location = reverseGeolocate(latitude=data_set['GPSLatitude'], longitude=data_set['GPSLongitude'], map_type=map_type)
maps_location = reverseGeolocate(
latitude=data_set['GPSLatitude'],
longitude=data_set['GPSLongitude'],
map_type=map_type
)
# cache data with Lat/Long
data_cache[cache_key] = maps_location
from_cache = False
@@ -1290,7 +1335,7 @@ for xmp_file in work_files: # noqa: C901
from_cache = True
# overwrite sets (note options check here)
if args.debug:
print("### Map Location ({}): {}".format(map_type, maps_location))
print(f"### Map Location ({map_type}): {maps_location}")
# must have at least the country set to write anything back
if maps_location['Country']:
for loc in data_set_loc:
@@ -1306,7 +1351,7 @@ for xmp_file in work_files: # noqa: C901
failed = True
else:
if args.debug:
print("Lightroom data use: {}, Lightroom data ok: {}".format(use_lightroom, lightroom_data_ok))
print(f"Lightroom data use: {use_lightroom}, Lightroom data ok: {lightroom_data_ok}")
# check if the data_set differs from the original (LR db load)
# if yes write, else skip
if use_lightroom and lightroom_data_ok:
@@ -1350,25 +1395,32 @@ if use_lightroom:
lrdb.close()
# end stats only if we write
print("{}".format('=' * 40))
print("XMP Files found : {:9,}".format(count['all']))
print(f"{'=' * 40}")
print(f"XMP Files found : {count['all']:9,}")
if args.read_only:
print("XMP Files listed : {:9,}".format(count['listed']))
print(f"XMP Files listed : {count['listed']:9,}")
if not args.read_only:
print("Updated : {:9,}".format(count['changed']))
print("Skipped : {:9,}".format(count['skipped']))
print("New GeoLocation from Map : {:9,}".format(count['map']))
print("GeoLocation from Cache : {:9,}".format(count['cache']))
print("GeoLocation from Fuzzy Cache : {:9,}".format(count['fuzzy_cache']))
print("Failed reverse GeoLocate : {:9,}".format(count['failed']))
print(f"Updated : {count['changed']:9,}")
print(f"Skipped : {count['skipped']:9,}")
print(f"New GeoLocation from Map : {count['map']:9,}")
print(f"GeoLocation from Cache : {count['cache']:9,}")
print(f"GeoLocation from Fuzzy Cache : {count['fuzzy_cache']:9,}")
print(f"Failed reverse GeoLocate : {count['failed']:9,}")
if use_lightroom:
print("GeoLocaction from Lightroom : {:9,}".format(count['lightroom']))
print("No Lightroom data found : {:9,}".format(count['not_found']))
print("More than one found in LR : {:9,}".format(count['many_found']))
print(f"GeoLocaction from Lightroom : {count['lightroom']:9,}")
print(f"No Lightroom data found : {count['not_found']:9,}")
print(f"More than one found in LR : {count['many_found']:9,}")
# if we have failed data
if len(failed_files) > 0:
print("{}".format('-' * 40))
print(f"{'-' * 40}")
print("Files that failed to update:")
print("{}".format(', '.join(failed_files)))
print(f"{', '.join(failed_files)}")
##############################################################
# MAIN RUN
##############################################################
main()
# __END__

8
requirement.txt Normal file
View File

@@ -0,0 +1,8 @@
certifi==2022.9.24
charset-normalizer==2.1.1
idna==3.4
install==1.3.5
python-xmp-toolkit==2.0.1
pytz==2022.6
requests==2.28.1
urllib3==1.26.12