Move the convert config setting to config other

the convert constant setting for inmage magick is now in config.other as
it is no longer core

Also add the base Progress and functions perl modules for central
tagging.

We will add config.pm, import_functions.pm, layout.pm and a basic test
script later too

4dev/database/update/edit_tables_missing_columns.sql

@@ -0,0 +1,14 @@
+-- update missing edit_* table data
+
+ALTER TABLE edit_generic ADD cuid VARCHAR;
+
+ALTER TABLE edit_access ADD enabled SMALLINT DEFAULT 0;
+ALTER TABLE edit_access ADD protected SMALLINT DEFAULT 0;
+
+ALTER TABLE edit_group ADD uid VARCHAR;
+ALTER TABLE edit_group ADD deleted SMALLINT DEFAULT 0;
+
+ALTER TABLE temp_files ADD folder varchar;
+ALTER TABLE edit_page ADD hostname varchar;
+
+ALTER TABLE edit_user ADD deleted SMALLINT DEFAULT 0;

4dev/update/20190910_page_content/20190910_page_content_updates.sql

@@ -88,3 +88,5 @@ UPDATE edit_query_string SET cuid = random_string(12) WHERE cuid IS NULL;
 UPDATE edit_scheme SET cuid = random_string(12) WHERE cuid IS NULL;
 UPDATE edit_user SET cuid = random_string(12) WHERE cuid IS NULL;
 UPDATE edit_visible_group SET cuid = random_string(12) WHERE cuid IS NULL;
+
+-- update all triggers

bin/Progress.pm

@@ -0,0 +1,498 @@
+package Progress;
+
+# AUTHOR: Clemens Schwaighofer
+# DATE CREATED: 2009/6/16
+# DESCRIPTION: progress percent class
+
+# METHODS
+# * init
+# my $prg = Progress->new();
+#   will init a new progress class in the var $prg
+#   the following parameters can be set directly during a new call
+#   - verbose (1/0)
+#   - precision (-1~10)
+#   - wide_time (0/1)
+#   - microtime (0/1)
+#   setting is done via
+# my $prg = Progress->new(verbose => 1, microtime = 1);
+# * setting methods
+# verbose($level int)
+#   $level has to be int, if not set there is no output show, at least 1 has to be given to see visible output
+# precision($decimals int)
+#   $decimals has to be int, if set to -1 then the steps are done in 10 increase, else it sets how many decimals are visible, 0 for no decimals
+# wide_time(0/1 int)
+#   sets the flag for wide time, if set to 1 the estimated time to end and time run is left prefixed with 15 chars
+# microtime(0/1 int)
+#   sets the flag to always show microtime (1) or only if the previous time was the same (0)
+# reset()
+#   resets all the internal vars for another new run
+# SetStartTime(optional timestamp)
+#   sets the start times for this progress run, the overall start/end time is set, and the time used for the actual progress
+#   in case there is some processing done before the run starts, it is highly recommended to call SetETAStartTime before the actual processing starts
+#   if no timestamp is given, internal timestamp is used (this is recommended)
+# SetETAStartTime(optional timestamp)
+#   only sets the start/end time for the actual "estimated time" calculation. It is recommended to call this right before the processing loop starts
+#   eg if there is a big query running that takes a lot of time, this method should be called before the reading loop
+#   as with SetStartTime a timestamp can be given, if not then the internal timestamp is used (this is recommended)
+# SetEndTime(optional timestamp)
+#   sets the end time for the overall processing. This should be called at the very end of the script before any final stat data is printed
+# linecount($lines int)
+#   sets the maximum lines that will be processed, used for percentage calculation. If non int is given, will set to 1. This will be only set once, to
+#   reset used reset() method.
+#   Either this or filesize NEED to be set
+# filesize($bytes int)
+#   filesize in bytes, if non valid data is given, then it is set to 1.
+#   filesize() and linecount() can both be set, but at least one of them has to be set.
+#   if filesize is set a byte data output is added, if only linecount is given, only the linecount output will be given (no bytes per second, etc)
+# ShowPosition(optional current byte position int)
+#   this is the main processing and has to be called at the end of the loop where the data is processed. If no bytes are given the internal counter (linecount)
+#   is used.
+#   for bytes it is recommended to use IO::File and $FH->tell to pass on the bytes
+#
+# VARIABLES
+# * internal set
+#   change: flagged 1 if output is given or would be given. can be used for any post processing after the ShowPosition is called
+#   precision_ten_step: flagged 1 if the precision was set to -1
+#   start: overall start time
+#   end: overall end time
+#   count: count of processed lines
+#   [TODO: describe the others too, at the moment only below in %fields]
+
+use strict;
+use warnings;
+use utf8;
+
+BEGIN {
+	use POSIX;
+	use Carp;
+	use Time::HiRes qw(time);
+	use File::Basename;
+	use Number::Format qw(format_number);
+	use vars qw($AUTOLOAD);
+	push(@INC, File::Basename::dirname($0).'/');
+}
+
+# important includes
+use functions;
+
+# variable declarationf or access
+# * can be set
+# = only for read
+# unmarked are internal only, but can be read if they are needed in further processing in the script
+my %fields = (
+	linecount => 0, # * max lines in input
+	filesize => 0, # * max file size
+	precision => 1, # * comma after percent
+	wide_time => 0, # * if flagged 1, then the wide 15 char left bound format is used
+	verbose => 0, # * verbose status from outside
+	microtime => 0, # * microtime output for last run time (1 for enable, 0 for auto, -1 for disable)
+	change => 0, # = flag if output was given
+	start => undef, # = global start for the full script running time
+	start_run => undef, # = for the eta time, can be set after a query or long read in, to not create a wrong ETA time
+	start_time => undef, # loop start
+	end => undef, # = global end
+	end_time => undef, # loop end
+	count_size => undef, # = filesize current
+	count => 0, # = position current
+	current_count => 0, # last count (position)
+	lines_processed => 0, # lines processed in the last run
+	last_group => 0, # time in seconds for the last group run (until percent change)
+	lines_in_last_group => 0, # float value, lines processed per second to the last group run
+	lines_in_global => 0, # float values, lines processed per second to complete run
+	bytes_in_last_group => 0, # flaot value, bytes processes per second in the last group run
+	bytes_in_global => 0, # float value, bytes processed per second to complete run
+	size_in_last_group => 0, # bytes processed in last run (in bytes)
+	current_size => 0, # current file position (size)
+	last_percent => 0, # last percent position
+	precision_ten_step => 0, # if we have normal % or in steps of 10
+	percent_print => 5, # the default size, this is precision + 4
+	percent_precision => 1, # this is 1 if it is 1 or 0 for precision, or precision size
+	eta => undef, # estimated time to finish
+	full_time_needed => undef, # run time since start
+	lg_microtime => 0 # last group microtime, this is auto set during process.
+);
+
+# class init
+sub new
+{
+	my $proto = shift;
+	my $class = ref($proto) || $proto;
+	my %data = @_;
+	my $self = {
+		_permitted => \%fields,
+		%fields,
+	};
+	# vars to init
+	bless ($self, $class);
+	if ($data{'verbose'} && $data{'verbose'} =~ /^\d{1}$/) {
+		$self->{verbose} = $data{'verbose'};
+	}
+	if (exists($data{'precision'}) && (($data{'precision'} || $data{'precision'} == 0) && $data{'precision'} =~ /^\-?\d{1,2}$/)) {
+		$self->precision($data{'precision'});
+	}
+	if ($data{'microtime'} && $data{'microtime'} =~ /^(0|1)$/) {
+		$self->microtime($data{'microtime'});
+	}
+	if ($data{'wide_time'} && $data{'wide_time'} =~ /^(0|1)$/) {
+		$self->wide_time($data{'wide_time'});
+	}
+	return $self;
+}
+
+# auto load for vars
+sub AUTOLOAD
+{
+	my $self = shift;
+	my $type = ref($self) || croak "$self is not an object";
+	my $name = $AUTOLOAD;
+	$name =~ s/.*://;
+
+	unless (exists $self->{_permitted}->{$name}) {
+		croak "Can't access '$name' field in class $type";
+	}
+
+	if (@_) {
+		return $self->{$name} = shift;
+	} else {
+		return $self->{$name};
+	}
+}
+
+# destructor
+sub DESTROY
+{
+	# do nothing, there is nothing to close or finish
+}
+
+# SUB: reset
+# PARAMS: none
+# DESC: resets all the current counters only and current start times
+sub reset
+{
+	my $self = shift;
+	# reset what always gets reset
+	$self->{count} = 0;
+	$self->{count_size} = undef;
+	$self->{current_count} = 0;
+	$self->{linecount} = 0;
+	$self->{lines_processed} = 0;
+	$self->{last_group} = 0;
+	$self->{lines_in_last_group} = 0;
+	$self->{lines_in_global} = 0;
+	$self->{bytes_in_last_group} = 0;
+	$self->{bytes_in_global} = 0;
+	$self->{size_in_last_group} = 0;
+	$self->{filesize} = 0;
+	$self->{current_size} = 0;
+	$self->{last_percent} = 0;
+	$self->{eta} = 0;
+	$self->{full_time_needed} = 0;
+	$self->{start_run} = undef;
+	$self->{start_time} = undef;
+	$self->{end_time} = undef;
+}
+
+# SUB: microtime
+# PARAMS: 1/0
+# DESC: flag to set microtime on or off in the time output
+#       if not 1 or 0, set to 0
+sub microtime
+{
+	my $self = shift;
+	my $microtime;
+	if (@_) {
+		$microtime = shift;
+		if ($microtime == 1 || $microtime == 0) {
+			$self->{microtime} = $microtime;
+		} else {
+			$self->{microtime} = 0;
+		}
+	}
+	return $self->{microtime};
+}
+
+
+# SUB: wide_time
+# PARAMS: 1/0
+# DESC: flag to set wide_time (15 char spacer).
+#       if not 1 or 0, set to 0
+sub wide_time
+{
+	my $self = shift;
+	my $wide;
+	if (@_) {
+		$wide = shift;
+		if ($wide == 1 || $wide == 0) {
+			$self->{wide_time} = $wide;
+		} else {
+			$self->{wide_time} = 0;
+		}
+	}
+	return $self->{wide_time};
+}
+
+# SUB: precision
+# PARAMS: precision in int
+# DESC: sets the output percent precision calculation and printf width
+#       if negative, to ten step, if bigger 10, set to one
+sub precision
+{
+	my $self = shift;
+	my $comma;
+	if (@_) {
+		$comma = shift;
+		$comma = 0 if ($comma !~ /^\-?\d{1,}$/);
+		if ($comma < 0) {
+			# -2 is 5 step
+			# -1 is 10 step
+			if ($comma < -1) {
+				$self->{precision_ten_step} = 5;
+			} else {
+				$self->{precision_ten_step} = 10;
+			}
+			$self->{precision} = 0; # no comma
+			$self->{percent_precision} = 0; # no print precision
+			$self->{percent_print} = 3; # max 3 length
+		} else {
+			$self->{precision} = $comma < 0 || $comma > 10 ? 10 : $comma;
+			$self->{percent_precision} = $comma < 0 || $comma > 10 ? 10 : $comma;
+			$self->{percent_print} = ($comma == 0 ? 3 : 4) + $self->{percent_precision};
+		}
+	}
+	return $self->{precision};
+}
+
+# SUB: linecount
+# PARAMS: max number of lines to be processed
+# DESC: sets the max number for lines for the percent calculation, if negative or not number, set to 1
+#       can only be set ONCE
+sub linecount
+{
+	my $self = shift;
+	my $linecount;
+	if (!$self->{linecount}) {
+		if (@_) {
+			$linecount = shift;
+			$self->{linecount} = $linecount;
+			$self->{linecount} = 1 if ($linecount < 0 || $linecount !~ /\d+/)
+		}
+	}
+	return $self->{linecount};
+}
+
+# SUB: filesize
+# PARAMS: max filesize for the to processed data
+# DESC: sets the max filesize for the to processed data, if negative or not number, set to 1
+#       input data has to be in bytes without any suffix (no b, kb, etc)
+#       can only be set ONCE
+sub filesize
+{
+	my $self = shift;
+	my $filesize;
+	if (!$self->{filesize}) {
+		if (@_) {
+			$filesize = shift;
+			$self->{filesize} = $filesize;
+			$self->{filesize} = 1 if ($filesize < 0 || $filesize !~ /\d+/)
+		}
+	}
+	return $self->{filesize};
+}
+
+# SUB: SetStartTime
+# PARAMS: time, or nothing
+# DESC: sets all the start times
+sub SetStartTime
+{
+	my $self = shift;
+	if (@_) {
+		$self->{start} = shift;
+	} else {
+		$self->{start} = time();
+	}
+	$self->{start_time} = $self->{start};
+	$self->{start_run} = $self->{start};
+}
+
+# SUB: SetETAStartTime
+# PARAMS: time, or nothing
+# DESC: sets the loop & run time, for correct ETA callculation
+sub SetETAStartTime
+{
+	my $self = shift;
+	if (@_) {
+		$self->{start_time} = shift;
+	} else {
+		$self->{start_time} = time();
+	}
+	$self->{start_run} = $self->{start_time};
+}
+
+# SUB: SetEndTime
+# PARAMS: time, or nothing
+# DESC: sets the end time for running time calculation
+sub SetEndTime
+{
+	my $self = shift;
+	if (@_) {
+		$self->{end} = shift;
+	} else {
+		$self->{end} = time();
+	}
+}
+
+# SUB: ShowPosition
+# PARAMS: optiona; file position (via file pointer)
+# RETURN: string for percent position output
+# DESC: calculates the current percent position based on the passed parameter, if no parameter uses intneral counter
+sub ShowPosition
+{
+	my $self = shift;
+	# set local vars
+	my $percent; # current percent
+	my $full_time_needed; # complete process time
+	my $full_time_per_line; # time per line
+	my $eta; # estimated end time
+	my $string = ''; # percent string that gets output
+	my $show_filesize = 1;
+	# microtime flags
+	my $eta_microtime = 0;
+	my $ftn_microtime = 0;
+	my $lg_microtime = 0;
+	# percent precision calc
+	my $_p_spf = "%.".$self->{precision}."f";
+	# output format for percent
+	my $_pr_p_spf = "%".$self->{percent_print}.".".$self->{percent_precision}."f";
+	# set the linecount precision based on the final linecount, if not, leave it empty
+	my $_pr_lc = "%s";
+	$_pr_lc = "%".length(format_number($self->{linecount}))."s" if ($self->{linecount});
+	# time format, if flag is set, the wide format is used
+	my $_pr_tf = "%s";
+	$_pr_tf = "%-15s" if ($self->{'wide_time'});
+	# do the smae for file size
+	# my $_pr_fs = "%s";
+	# $_pr_fs = "%".length(function::convert_number($self->{filesize}))."s" if ($self->{filesize});
+
+	# increase position by one
+	$self->{count} ++;
+	# see if we get anything from IO tell
+	if (@_) {
+		$self->{file_pos} = shift;
+	} else {
+		# we did not, so we set internal value
+		$self->{file_pos} = $self->{count};
+		# we also check if the filesize was set now
+		if (!$self->{filesize}) {
+			$self->{filesize} = $self->{linecount};
+		}
+		# set ignore filesize output (no data)
+		$show_filesize = 0;
+	}
+	# set the count size based on the file pos, is only used if we have filesize
+	$self->{count_size} = $self->{file_pos};
+
+	# do normal or down to 10 (0, 10, ...) %
+	if ($self->{precision_ten_step}) {
+		# calc 0 comma precision, so just do a floor
+		my $_percent = sprintf("%d", ($self->{file_pos} / $self->{filesize}) * 100);
+		# mod that to 10
+		my $mod = $_percent % $self->{precision_ten_step};
+		# either write this one, or write the previous, old one
+		$percent = $mod == 0 ? $_percent : $self->last_percent;
+		# print "P: $percent, Last: ".$self->last_percent.", Mod: ".$mod.", Calc: ".$_percent."\n";
+	} else {
+		$percent = sprintf($_p_spf, ($self->{file_pos} / $self->{filesize}) * 100);
+	}
+	# print "POS: ".$self->{file_pos}.", PERCENT: $percent / ".$self->last_percent."\n";
+	if ($percent != $self->last_percent) {
+		$self->{end_time} = time();
+		# for from the beginning
+		$full_time_needed = $self->{end_time} - $self->{start_run}; # how long from the start;
+		$self->{last_group} = $self->{end_time} - $self->{start_time};
+		$self->{lines_processed} = $self->{count} - $self->{current_count};
+		# lines in last group
+		$self->{lines_in_last_group} = $self->{'last_group'} ? ($self->{lines_processed} / $self->{last_group}) : 0;
+		# lines in global
+		$self->{lines_in_global} = $full_time_needed ? ($self->{'count'} / $full_time_needed) : 0;
+		# if we have linecount
+		if (!$self->{linecount}) {
+			$full_time_per_line = (($full_time_needed) ? $full_time_needed : 1) / $self->{count_size}; # how long for all
+			$eta = $full_time_per_line * ($self->{filesize} - $self->{count_size}); # estimate for the rest
+		} else {
+			$full_time_per_line = (($full_time_needed) ? $full_time_needed : 1) / $self->{count}; # how long for all
+			$eta = $full_time_per_line * ($self->{linecount} - $self->{count}); # estimate for the rest
+		}
+
+		# just in case ...
+		$eta = '0' if ($eta < 0);
+		# check if to show microtime
+		# ON: if microtime is flagged as one
+		$eta_microtime = $ftn_microtime = $lg_microtime = 1 if ($self->{microtime} == 1);
+		# AUTO: foir microtime
+		if ($self->{microtime} == 0) {
+			$eta_microtime = 1 if ($eta > 0 && $eta < 1);
+			$ftn_microtime = 1 if ($full_time_needed > 0 && $full_time_needed < 1);
+			# pre check last group: if pre comma part is same add microtime anyway
+			$lg_microtime = 1 if ($self->{last_group} > 0 && $self->{last_group} < 1);
+		}
+		# print out
+		if ($show_filesize) {
+			# last group size
+			$self->{size_in_last_group} = $self->{count_size} - $self->{current_size};
+			# calc kb/s if there is any filesize data
+			# last group
+			$self->{bytes_in_last_group} = $self->{'last_group'} ? ($self->{size_in_last_group} / $self->{last_group}) : 0;
+			# global
+			$self->{bytes_in_global} = $full_time_needed ? ($self->{count_size} / $full_time_needed) : 0;
+			# only used if we run with file size for the next check
+			$self->{current_size} = $self->{count_size};
+
+			$string = sprintf(
+				"Processed ".$_pr_p_spf."%% [%s / %s] | ".$_pr_lc." / ".$_pr_lc." Lines | ETA: ".$_pr_tf." / TR: ".$_pr_tf." / LR: %s lines (%s) in %s, %s (%s) lines/s, %s (%s) b/s\n",
+				$percent,
+				function::convert_number($self->{count_size}),
+				function::convert_number($self->{filesize}),
+				format_number($self->{count}),
+				format_number($self->{linecount}),
+				function::convert_time($eta, $eta_microtime),
+				function::convert_time($full_time_needed, $ftn_microtime),
+				format_number($self->{lines_processed}),
+				function::convert_number($self->{size_in_last_group}),
+				function::convert_time($self->{last_group}, $lg_microtime),
+				format_number($self->{lines_in_global}, 2, 1),
+				format_number($self->{lines_in_last_group}, 2, 1),
+				function::convert_number($self->{bytes_in_global}),
+				function::convert_number($self->{bytes_in_last_group})
+			) if ($self->{verbose} >= 1);
+		} else {
+			$string = sprintf(
+				"Processed ".$_pr_p_spf."%% | ".$_pr_lc." / ".$_pr_lc." Lines | ETA: ".$_pr_tf." / TR: ".$_pr_tf." / LR: %s lines in %s, %s (%s) lines/s\n",
+				$percent,
+				format_number($self->{count}),
+				format_number($self->{linecount}),
+				function::convert_time($eta, $eta_microtime),
+				function::convert_time($full_time_needed, $ftn_microtime),
+				format_number($self->{lines_processed}),
+				function::convert_time($self->{last_group}, $lg_microtime),
+				format_number($self->{lines_in_global}, 2, 1),
+				format_number($self->{lines_in_last_group}, 2, 1)
+			) if ($self->{verbose} >= 1);
+		}
+		# write back vars
+		$self->{last_percent} = $percent;
+		$self->{eta} = $eta;
+		$self->{full_time_needed} = $full_time_needed;
+		$self->{lg_microtime} = $lg_microtime;
+		# for the next run, check data
+		$self->{start_time} = time();
+		$self->{current_count} = $self->{count};
+		# trigger if this is a change
+		$self->{change} = 1;
+	} else {
+		# trigger if this is a change
+		$self->{change} = 0;
+	}
+	return $string;
+}
+
+1;

bin/functions.pm

@@ -0,0 +1,501 @@
+package function;
+
+# AUTHOR: Clemens Schwaighofer
+# DATE CREATED: 2004/11/09
+# DESCRIPTION: functions collection for Adidas scripts
+# HISTORY:
+# 2005/06/22 (cs) added header key check function
+# 2005/02/10 (cs) added debug flag to print output, added two new functions to format a number into B, KB, etc
+# 2005/01/13 (cs) fixed array problem with the clean up and int function
+
+use strict;
+use warnings;
+use 5.000_000;
+use POSIX qw(floor);
+use File::Copy;
+use Digest::SHA qw(sha1_hex);
+use utf8;
+#require Exporter;
+#our @ISA = qw(Exporter);
+#our @EXPORT = qw();
+
+# depending on the options given to the program, it gets the correct settings
+# to which db it should connect
+sub get_db_user
+{
+	my ($target, $db) = @_;
+
+	# the parts of the hash array (tab seperated)
+	my @array_names = qw{db_name	db_port	db_user	db_pass	db_host	db_type	db_test	db_ssl};
+	my %db_out = ();
+
+	# based on the two parameters find the correct vars
+	# each level can hold data, higher level data overrules lower data
+	# eg $config::db{'test'}{'db_user'} overrules $config::db{'db_user'}
+	for (my $i = 1; $i <= 3; $i ++) {
+		foreach my $name (@array_names) {
+			# depending on the level check the level of data
+			if ($i == 1) {
+				$db_out{$name} = $config::db{$name} if (defined($config::db{$name}));
+			} elsif ($i == 2) {
+				$db_out{$name} = $config::db{$target}{$name} if (defined($config::db{$target}{$name}));
+			} elsif ($i == 3) {
+				$db_out{$name} = $config::db{$target}{$db}{$name} if (defined($config::db{$target}{$db}{$name}));
+			}
+		} # for each db data var
+	} # for each data level in the hash
+
+	return (
+		$db_out{'db_name'},
+		$db_out{'db_port'},
+		$db_out{'db_user'},
+		$db_out{'db_pass'},
+		$db_out{'db_host'},
+		$db_out{'db_type'},
+		$db_out{'db_test'},
+		$db_out{'db_ssl'}
+	);
+}
+
+# get the DSN string for the DB connect
+sub get_db_dsn
+{
+	my (
+		$db_name,
+		$db_port,
+		$db_user,
+		$db_pass,
+		$db_host,
+		$db_type,
+		$db_ssl
+	) = @_;
+	my $dsn = '';
+
+	if ($db_type eq 'mysql' && $db_name && $db_host && $db_user) {
+		$dsn = "DBI:mysql:database=".$db_name.";host=".$db_host.";port=".$db_port;
+	} elsif ($db_type eq 'pgsql' && $db_name && $db_host && $db_user) {
+		$dsn = "DBI:Pg:dbname=".$db_name.";host=".$db_host.";port=".$db_port.";sslmode=".$db_ssl;
+	} else {
+		# invalid db type
+		$dsn = -1;
+	}
+	return $dsn;
+}
+
+sub strip_white_spaces
+{
+	my ($element) = @_;
+	# get rid of spaces at the end and at the beginning of each bloack
+	$element =~ s/^\s+//g;
+	$element =~ s/\s+$//g;
+	return $element;
+}
+
+sub prepare_hash_keys
+{
+	my($csv, $data, $csv_header) = @_;
+
+	# unset value starts at 1000 and goes up ...
+	my $unset_value = 1000;
+	my %keys = ();
+
+	# parse header
+	if ($csv->parse($data)) {
+		my @cols = $csv->fields();
+		for (my $i = 0; $i < @cols; $i ++) {
+			# remove all spaces before and afterward
+			$cols[$i] = function::strip_white_spaces($cols[$i]);
+			# write key - id number
+			$keys{$cols[$i]} = $i;
+			print $::DEBUG "\tPostion [".$i."]: ".$cols[$i]."\n" if ($::debug);
+			print "\tPosition [".$i."]: ".$cols[$i]."\n" if ($::verbose > 1);
+		}
+	} else {
+		die "ERROR[".$csv->error_diag()."]: ".$csv->error_input()."\n";
+	}
+	# add empty values
+	foreach my $csv_header_value (@$csv_header) {
+		if (!defined($keys{$csv_header_value})) {
+			$keys{$csv_header_value} = $unset_value;
+			$unset_value ++;
+			print $::DEBUG "\tKey [$csv_header_value] gets position [".$keys{$csv_header_value}."]\n" if ($::debug);
+			print "\tKey [$csv_header_value] gets position [".$keys{$csv_header_value}."]\n" if ($::verbose > 1);
+		}
+	}
+
+	return %keys;
+}
+
+sub error_check_keys
+{
+	my($csv_header, $keys) = @_;
+
+	if ((keys %$keys) != @$csv_header) {
+		print $::ERR "TOTAL WRONG COUNT: CSV header ".(keys %$keys)." vs Needed headers ".@$csv_header.": perhaps your input file is not fitting this?\n";
+		print "TOTAL WRONG COUNT: CSV header ".(keys %$keys)." vs Needed headers ".@$csv_header.": perhaps your input file is not fitting this?\n";
+
+		# if there are more keys in CSV file, then in the header defined in here
+		if ((keys %$keys) > @$csv_header) {
+			print $::ERR "Listing Perl Header missing\n";
+			print "Listing Perl Header missing\n";
+			foreach my $key (keys %$keys) {
+				print $::ERR "Missing in perl Header list: $key\n" if (!grep {$_ eq $key} @$csv_header);
+				print "Missing in perl Header list: $key\n" if (!grep {$_ eq $key} @$csv_header);
+			}
+			# if more keys are in the header defined than in the csv file
+		} else {
+			print $::ERR "Listing CSV Header missing\n";
+			print "Listing CSV Header missing\n";
+			for (my $i = 0; $i < @$csv_header; $i ++) {
+				print $::ERR "Missing in CSV file: ".$$csv_header[$i]."\n" if (!defined($$keys{$$csv_header[$i]}));
+				print "Missing in CSV file: ".$$csv_header[$i]."\n" if (!defined($$keys{$$csv_header[$i]}));
+			}
+		}
+		return 0;
+	}
+	return 1;
+}
+
+sub clean_up_row
+{
+	my ($row) = @_;
+
+	for (my $i = 0; $i < @$row; $i++) {
+		# get rid of spaces at the end and at the beginning of each bloack
+		$$row[$i] =~ s/^\s+//g;
+		$$row[$i] =~ s/\s+$//g;
+		# convert all half width Katakan to Full width Katakana
+		$$row[$i] = Unicode::Japanese->new($$row[$i])->h2zKana->get;
+		# need to decode the converted string, somehow Unicode::Japanese does not return proper utf8 if use utf8 is on
+		utf8::decode($$row[$i]);
+	}
+
+	return @$row;
+}
+
+sub set_int_fields
+{
+	my ($row, $keys, $int_fields) = @_;
+
+	# check ALL smallint/int/etc rows to be set to a number
+	for (my $i = 0; $i < @$int_fields; $i++) {
+		print "\t\tCheck ".$$int_fields[$i]." {".$$keys{$$int_fields[$i]}."} ... " if ($::verbose > 1);
+		if (!$$row[$$keys{$$int_fields[$i]}]) {
+			$$row[$$keys{$$int_fields[$i]}] = 0;
+		}
+		# if its filled, but not a digit, set to 1
+		if ($$row[$$keys{$$int_fields[$i]}] =~ /\D/) {
+			$$row[$$keys{$$int_fields[$i]}] = 1;
+		}
+		print "[".$$row[$$keys{$$int_fields[$i]}]."] [DONE]\n" if ($::verbose > 1);
+	}
+	return @$row;
+}
+
+# formats a number with dots and ,
+sub format_number
+{
+	my ($number) = @_;
+	# dummy, does nothing now
+	# should put . or , every 3 digits later
+	return $number;
+}
+
+# converts bytes to human readable format
+sub convert_number
+{
+	my ($number) = @_;
+	my $pos; # the original position in the labels array
+	# divied number until its division would be < 1024. count that position for label usage
+	for ($pos = 0; $number > 1024; $pos ++) {
+		$number = $number / 1024;
+	}
+	# before we return it, we format it [rounded to 2 digits, if has decimals, else just int]
+	# we add the right label to it and return
+	return sprintf(!$pos ? '%d' : '%.2f', $number)." ".qw(B KB MB GB TB PB EB)[$pos];
+}
+
+# make time from seconds string
+sub convert_time
+{
+	my ($timestamp, $show_micro) = @_;
+	my $ms = '';
+	# cut of the ms, but first round them up to four
+	$timestamp = sprintf("%.4f", $timestamp);
+	# print "T: ".$timestamp."\n";
+	($timestamp, $ms) = split(/\./, $timestamp);
+	my @timegroups = ("86400", "3600", "60", "1");
+	my @output = ();
+	for (my $i = 0; $i < @timegroups; $i ++) {
+		push(@output, floor($timestamp / $timegroups[$i]));
+		$timestamp = $timestamp % $timegroups[$i];
+	}
+	# output has days|hours|min|sec
+	return (($output[0]) ? $output[0]."d " : "").
+		(($output[1] || $output[0]) ? $output[1]."h " : "").
+		(($output[2] ||$output[1] || $output[0]) ? $output[2]."m " : "").
+		$output[3]."s".
+		(($show_micro) ? " ".((!$ms) ? 0 : $ms)."ms" : "");
+}
+
+# get a timestamp and create a proper formated date/time field
+sub create_time
+{
+	my ($timestamp, $show_micro) = @_;
+	my $ms = '';
+	$timestamp = 0 if (!$timestamp);
+	# round ms to 4 numbers
+	$timestamp = sprintf("%.4f", $timestamp);
+	($timestamp, $ms) = split(/\./, $timestamp);
+	# array for time
+	my ($sec, $min, $hour, $day, $month, $year, $wday, $yday, $isdst) = localtime($timestamp);
+	# year, month fix
+	$year += 1900;
+	$month += 1;
+	# string for return
+	return $year."-".
+		($month < 10 ? '0'.$month : $month)."-".
+		($day < 10 ? '0'.$day : $day)." ".
+		($hour < 10 ? '0'.$hour : $hour).":".
+		($min < 10 ? '0'.$min : $min).":".
+		($sec < 10 ? '0'.$sec : $sec).
+		(($ms && $show_micro) ? ".".$ms : "");
+}
+
+# create YYYYMMDD data
+sub create_date
+{
+	my ($timestamp, $split_string) = @_;
+	my $split = $split_string ? $split_string : '';
+	$timestamp = time() if (!$timestamp);
+	# array for time
+	my ($sec, $min, $hour, $day, $month, $year, $wday, $yday, $isdst) = localtime($timestamp);
+	# year, month fix
+	$year += 1900;
+	$month += 1;
+	# string for return
+	return $year.$split.
+		($month < 10 ? '0'.$month : $month).$split.
+		($day < 10 ? '0'.$day : $day);
+}
+
+# create YYYYMMDD_HHMMSS data
+sub create_datetime
+{
+	my ($timestamp, $split_string) = @_;
+	my $split = $split_string ? $split_string : '';
+	$timestamp = time() if (!$timestamp);
+	# array for time
+	my ($sec, $min, $hour, $day, $month, $year, $wday, $yday, $isdst) = localtime($timestamp);
+	# year, month fix
+	$year += 1900;
+	$month += 1;
+	# string for return
+	return $year.$split.
+		($month < 10 ? '0'.$month : $month).$split.
+		($day < 10 ? '0'.$day : $day).'_'.
+		($hour < 10 ? '0'.$hour : $hour).$split.
+		($min < 10 ? '0'.$min : $min).$split.
+		($sec < 10 ? '0'.$sec : $sec);
+}
+
+sub left_fill
+{
+	my($number, $size, $char) = @_;
+	return sprintf($char x ($size - length($number)).$number);
+}
+
+# wrapper to flip the crc32 hex string, so it is like buggy php one (php <= 5.2.6)
+sub crc32b_fix
+{
+	my ($crc) = @_;
+	# left pad with 0 to 8 chars
+	$crc = ('0' x (8 - length($crc))).$crc;
+	# flip two chars (byte hex)
+	$crc =~ s/^([a-z0-9]{2})([a-z0-9]{2})([a-z0-9]{2})([a-z0-9]{2})$/$4$3$2$1/;
+	return $crc;
+}
+
+# short sha1 (9 char) function
+sub sha1_short
+{
+	my ($string) = @_;
+	return substr(sha1_hex($string), 0, 9);
+}
+
+# DEBUG helpers for dumping data
+# from: http://www.perlmonks.org/?node_id=390153
+# alternative use Dump::Dumper and print Dump(VAR);
+sub dump_data
+{
+	my ($level, $base, $data) = @_;
+	my $nextlevel = $level + 1;
+	if (ref($data) eq 'ARRAY') {
+		foreach my $k (0 .. $#{$data}) {
+			my $baseval = $base.'['.$k.']';
+			dump_it($nextlevel, $baseval, $data->[$k]);
+		}
+	} elsif (ref($data) eq 'HASH') {
+		foreach my $k (sort(keys(%{$data}))) {
+			my $baseval = $base.'{'.$k.'}';
+			dump_it($nextlevel, $baseval, $data->{$k});
+		}
+	} elsif (ref($data) eq 'SCALAR') {
+		my $baseval = $base;
+		dump_it($nextlevel, $baseval, ${$data});
+	}
+}
+
+sub dump_it
+{
+	my ($nextlevel, $baseval, $datum) = @_;
+	my $reftype = ref($datum);
+	if ($reftype eq 'HASH') {
+		dump_data($nextlevel, $baseval, \%{$datum});
+	} elsif ($reftype eq 'ARRAY') {
+		dump_data($nextlevel, $baseval, \@{$datum});
+	} else {
+		process_data($nextlevel, $baseval, $datum);
+	}
+}
+
+sub process_data
+{
+	my ($nextlevel, $baseval, $datum) = @_;
+	my $indentation = '  ' x $nextlevel;
+	print $indentation, $baseval, ' = ', $datum, "\n";
+}
+
+# METHOD: lock_run
+# PARAMS: file (plus path) to lock to
+#         the current running pid (if not given will be set in script)
+#         the current name of the script (auto set if not given)
+#         optional write encoding (set to utf8 if not given)
+# RETURN: nothing
+# DESC:   checks if this script is already running based on the lock file, if if yes will abort
+#         if file is there but pid not find it automatically cleans up the stale lock file
+sub lock_run
+{
+	my ($file, $run_pid, $name, $encoding) = @_;
+	# if no encoding, set utf8
+	$encoding = 'utf8' if (!$encoding);
+	# set the run pid if no pid is given
+	$run_pid = $$ if (!$run_pid);
+	# set the script base name
+	$name = File::Basename::fileparse($0) if (!$name);
+	# if lock file exists
+	if (-f $file) {
+		my $exists = 0;
+		my $pid = `cat $file`;
+		chomp($pid);
+		# printDebug("Lock file found for $pid", 1);
+		# check if process excists with this pid
+		# better todo A for ALL processes
+		# ps axu OR short ps a
+		open(PS, 'ps axu|') || die("$!");
+		while (<PS>) {
+			# search for pid and run file name
+			if ($_ =~ /\ $pid\ / && $_ =~ /$name/) {
+				$exists = 1;
+			}
+			last if ($exists);
+		}
+		close(PS);
+		if (!$exists) {
+			# printDebug("Lock file cleaned up for $pid", 1);
+			unlink($file);
+		} else {
+			die("Script is already running with PID $pid\n");
+		}
+	}
+	# write current PID into lock file
+	open(FP, '>:encoding('.$encoding.')', $file) || die ("Cannot open run lock file '$file' for writing\n");
+	print FP $run_pid;
+	close(FP);
+}
+
+# METHOD: printDebug
+# PARAMS: message, verbose level
+# RETURN: nothing
+# DESC:   depeding on the verbose and debug settings it will print out message and or write it to a debug file
+sub printDebug
+{
+	my($msg, $vrb, $dbg) = @_;
+	# print debug only if debug is on and debug file is available
+	print $::DEBUG '['.create_time(time(), 1).'] '.$msg."\n" if ($::debug && $::DEBUG);
+	# print to log if log is accessable and the verbose flag matches, or for debug flag if debug statement is set and not log only, or if log only, if not debug statement
+	print $::LOG $msg."\n" if (($::verbose >= $vrb || (!$::log_only && $dbg && $::debug) || ($::log_only && !$dbg)) && $::LOG);
+	# print to screen if verbose matches, but it is not a log only, or if it is debug statement and debug flag is set
+	print $msg."\n" if (($::verbose >= $vrb && !$::log_only) || ($dbg && $::debug));
+}
+
+# METHOD: waitAbort
+# PARAMS: time in seconds, if not provided set to 5
+# RETURN: nothing
+# DESC:   simple prints out a char while waiting for an abort command
+sub waitAbort
+{
+	my($sleep) = @_;
+	$sleep = 5 if ($sleep !~ /\d/);
+	print "Waiting $sleep seconds (Press CTRL + C to abort)\n";
+	for (my $i = 1; $i <= $sleep; $i ++) {
+		print ".";
+		sleep 1;
+	}
+	print "\n\n";
+}
+
+# METHOD: copyToTemporary
+# PARAMS: file to copy, and target file name
+# RETURN: the target file name
+# DESC  : sets the source to read only and makes a copy, the copy is also set to read only
+sub copyToTemporary
+{
+	my ($source, $target) = @_;
+	# get the current rights
+	my $current_chmod = (stat $source)[2];
+	# set source file ARGV to read only
+	# we skip that, the source might be NOT from the same user as the script read, just copy the file and set the target read only
+	chmod(0444, $source);
+	# create tmp backup file from which we read, data gets removed at the end of an run, or during an abort call
+	copy($source, $target) || die("Copy failed: $!\n");
+	# set read rights to r only for the copied file
+	chmod(0444, $target);
+	# set old access rights for ARGV file
+	chmod($current_chmod, $source);
+	# return target file name
+	return $target;
+}
+
+# METHOD: uniq
+# PARAMS: @array
+# RETURN: array with only unique entries
+# DESC  : used in uniq(@array) to get only unique data back
+sub uniq
+{
+	my %seen;
+	grep !$seen{$_}++, @_;
+}
+
+# METHOD: clean_test
+# PARAMS: array of data
+# RETURN: cleaned up array of data
+# DESC  : sets all undefs to '' for debug output
+sub clean_test
+{
+	my (@data) = @_;
+	# map check for defined, if not, return ''
+	return map { defined($_) ? $_ : '' } @data;
+}
+
+# METHOD: clean_test_string
+# PARAMS: string to be checked
+# RETURN: data or empty for output
+# DESC  : sets all input data to '' if it is undefined
+sub clean_test_string
+{
+	my ($data) = @_;
+	return defined($data) ? $data : '';
+}
+
+1;

www/admin/class_test.php

@@ -254,6 +254,45 @@ echo "SOURCE ARRAY: ".$basic->printAr($test_array)."<br>";
 echo "FOUND ELEMENTS [base]: ".$basic->printAr($basic->arraySearchRecursive('email', $test_array, 'type'))."<br>";
 echo "FOUND ELEMENTS [input]: ".$basic->printAr($basic->arraySearchRecursive('email', $test_array['input'], 'type'))."<br>";
 
+// *** BYTES TEST ***
+$bytes = array(
+	-123123123,
+	999999, // KB-1
+	999999999, // MB-1
+	254779258, // MB-n
+	999999999999999, // TB-1
+	588795544887632, // TB-n
+	999999999999999999, // PB-1
+	9223372036854775807, // MAX INT
+	999999999999999999999, // EB-1
+);
+print "<b>BYTE FORMAT TESTS</b><br>";
+foreach ($bytes as $byte) {
+	print '<div style="display: flex; border-bottom: 1px dashed gray;">';
+	//
+	print '<div style="width: 35%; text-align: right; padding-right: 2px;">';
+	print "(".number_format($byte)."/".$byte.") bytes :";
+	print '</div><div style="width: 40%;">';
+	print $basic->humanReadableByteFormat($byte);
+	print "</div>";
+	//
+	print "</div>";
+	//
+	print '<div style="display: flex; border-bottom: 1px dotted red;">';
+	//
+	print '<div style="width: 35%; text-align: right; padding-right: 2px;">';
+	print "bytes [si]:";
+	print '</div><div style="width: 40%;">';
+	// print $basic->byteStringFormat($byte, true, false, true);
+	print $basic->humanReadableByteFormat($byte, $basic::BYTE_FORMAT_SI);
+	print "</div>";
+	//
+	print "</div>";
+}
+
+
+// *** IMAGE TESTS ***
+echo "<hr>";
 // image thumbnail
 $images = array(
 	// height bigger
@@ -272,7 +311,6 @@ $images = array(
 	// Photoshop
 	'photoshop_test.psd',
 );
-echo "<hr>";
 $thumb_width = 250;
 $thumb_height = 300;
 // return mime type ala mimetype

www/admin/namespace_test.php

@@ -21,8 +21,8 @@ if ($base->getConnectionStatus()) {
 }
 
 print "Start time: ".$base->runningTime()."<br>";
-print "ByteStringFormat: ".$base->ByteStringFormat(1234567.12)."<br>";
-print "byteStringFormat: ".$base->byteStringFormat(1234567.12)."<br>";
+print "HumanReadableByteFormat: ".$base->HumanReadableByteFormat(1234567.12)."<br>";
+print "humanReadableByteFormat: ".$base->humanReadableByteFormat(1234567.12)."<br>";
 // print "get_page_name [DEPRECATED]: ".$base->get_page_name()."<br>";
 print "getPageName: ".$base->getPageName()."<br>";
 

www/configs/config.master.php

@@ -54,6 +54,8 @@ DEFINE('PDFS', 'documents'.DS);
 DEFINE('CSV', 'csv'.DS);
 // css
 DEFINE('CSS', 'css'.DS);
+// font (web)
+DEFINE('FONT', 'font'.DS);
 // js
 DEFINE('JS', 'javascript'.DS);
 // table arrays
@@ -87,7 +89,7 @@ DEFINE('DEFAULT_ACL_LEVEL', 80);
 // 4: very strict, even on normal fixable errors through error
 // DEFINE('ERROR_STRICT', 3);
 // allow page caching in general, set to 'FALSE' if you do debugging or development!
-// DEFINE('ALLOW_SMARTY_CACHE', FALSE);
+// DEFINE('ALLOW_SMARTY_CACHE', false);
 // cache life time, in second', default here is 2 days (172800s)
 // -1 is never expire cache
 // DEFINE('SMARTY_CACHE_LIFETIME', -1);
@@ -112,6 +114,7 @@ DEFINE('USE_JQUERY', true);
 
 /************* LAYOUT WIDTHS *************/
 DEFINE('PAGE_WIDTH', 800);
+DEFINE('CONTENT_WIDTH', 800);
 // the default template name
 DEFINE('MASTER_TEMPLATE_NAME', 'main_body.tpl');
 
@@ -260,22 +263,6 @@ if (file_exists(BASE.CONFIGS.'config.other.php')) {
 	require BASE.CONFIGS.'config.other.php';
 }
 
-/************* CONVERT *******************/
-// this only needed if the external thumbnail create is used
-$paths = array(
-	'/bin',
-	'/usr/bin',
-	'/usr/local/bin'
-);
-// find convert
-foreach ($paths as $path) {
-	if (file_exists($path.DS.'convert') && is_file($path.DS.'convert')) {
-		// image magick convert location
-		DEFINE('CONVERT', $path.DS.'convert');
-	}
-}
-unset($paths);
-
 /************* DEBUG *******************/
 // turn off debug if debug flag is OFF
 if (defined('DEBUG') && DEBUG == false) {

www/configs/config.other.php

@@ -9,4 +9,20 @@
 
 // DEFINE('SOME_ID', <SOME VALUE>);
 
+/************* CONVERT *******************/
+// this only needed if the external thumbnail create is used
+$paths = array(
+	'/bin',
+	'/usr/bin',
+	'/usr/local/bin'
+);
+// find convert
+foreach ($paths as $path) {
+	if (file_exists($path.DS.'convert') && is_file($path.DS.'convert')) {
+		// image magick convert location
+		DEFINE('CONVERT', $path.DS.'convert');
+	}
+}
+unset($paths);
+
 // __END__

www/includes/edit_base.php

@@ -270,38 +270,57 @@ if ($form->my_page_name == 'edit_order') {
 
 	$position = 0;
 	$menu_data = array();
-	for ($i = 1; $i <= count($menuarray); $i ++) {
+	// for ($i = 1; $i <= count($menuarray); $i ++) {
+	foreach ($menuarray as $i => $data) {
 		// do that for new array
-		$j = $i - 1;
-		$menu_data[$j]['pagename'] = htmlentities($menuarray[($i-1)]['page_name']);
-		$menu_data[$j]['filename'] = $menuarray[($i-1)]['filename'].(isset($menuarray[$j]['query_string']) ? $menuarray[$j]['query_string'] : '');
-		if ($i == 1 || !($j % $SPLIT_FACTOR)) {
-			$menu_data[$j]['splitfactor_in'] = 1;
+		$j = $i + 1;
+		$menu_data[$i]['pagename'] = htmlentities($data['page_name']);
+		$menu_data[$i]['filename'] =
+			// prefix folder or host name
+			(isset($data['hostname']) && $data['hostname'] ?
+				$data['hostname'] :
+				''
+			).
+			// filename
+			$data['filename'].
+			// query string
+			(isset($data['query_string']) && $data['query_string'] ?
+				$data['query_string'] :
+				''
+			);
+		if ($j == 1 || !($i % $SPLIT_FACTOR)) {
+			$menu_data[$i]['splitfactor_in'] = 1;
 		} else {
-			$menu_data[$j]['splitfactor_in'] = 0;
+			$menu_data[$i]['splitfactor_in'] = 0;
 		}
-		if ($menuarray[$j]['filename'] == $form->getPageName()) {
+		// on matching, we also need to check if we are in the same folder
+		if ($data['filename'] == $form->getPageName() &&
+			(!isset($data['hostname']) || (
+				isset($data['hostname']) &&
+					(!$data['hostname'] || strstr($data['hostname'], CONTENT_PATH) !== false)
+			))
+		) {
 			$position = $j;
-			$menu_data[$j]['position'] = 1;
-			$menu_data[$j]['popup'] = 0;
+			$menu_data[$i]['position'] = 1;
+			$menu_data[$i]['popup'] = 0;
 		} else {
 			// add query stuff
 			// HAS TO DONE LATER ... set urlencode, etc ...
 			// check if popup needed
-			if (isset($menuarray[$j]['popup']) && $menuarray[$j]['popup'] == 1) {
-				$menu_data[$j]['popup'] = 1;
-				$menu_data[$j]['rand'] = uniqid((string)rand());
-				$menu_data[$j]['width'] = $menuarray[$j]['popup_x'];
-				$menu_data[$j]['height'] = $menuarray[$j]['popup_y'];
+			if (isset($data['popup']) && $data['popup'] == 1) {
+				$menu_data[$i]['popup'] = 1;
+				$menu_data[$i]['rand'] = uniqid((string)rand());
+				$menu_data[$i]['width'] = $data['popup_x'];
+				$menu_data[$i]['height'] = $data['popup_y'];
 			} else {
-				$menu_data[$j]['popup'] = 0;
+				$menu_data[$i]['popup'] = 0;
 			}
-			$menu_data[$j]['position'] = 0;
+			$menu_data[$i]['position'] = 0;
 		} // highlight or not
-		if (!($i % $SPLIT_FACTOR) || (($i + 1) > count($menuarray))) {
-			$menu_data[$j]['splitfactor_out'] = 1;
+		if (!($j % $SPLIT_FACTOR) || (($j + 1) > count($menuarray))) {
+			$menu_data[$i]['splitfactor_out'] = 1;
 		} else {
-			$menu_data[$j]['splitfactor_out'] = 0;
+			$menu_data[$i]['splitfactor_out'] = 0;
 		}
 	} // for
 	// $form->debug('MENU ARRAY', $form->printAr($menu_data));
@@ -357,14 +376,35 @@ if ($form->my_page_name == 'edit_order') {
 				if (!isset($form->table_array['edit_page_id']['value'])) {
 					$q = "DELETE FROM temp_files";
 					$form->dbExec($q);
-					// gets all files in the current dir ending with .php
-					$crap = exec('ls *.php', $output, $status);
-					// now get all that are NOT in de DB
-					$q = "INSERT INTO temp_files VALUES ";
-					for ($i = 0; $i < count($output); $i ++) {
-						$t_q = "('".$form->dbEscapeString($output[$i])."')";
-						$form->dbExec($q.$t_q, 'NULL');
+					// gets all files in the current dir and dirs given ending with .php
+					$folders = array('../admin/', '../frontend/');
+					$files = array('*.php');
+					$search_glob = array();
+					foreach ($folders as $folder) {
+						// make sure this folder actually exists
+						if (is_dir(ROOT.$folder)) {
+							foreach ($files as $file) {
+								$search_glob[] = $folder.$file;
+							}
+						}
 					}
+					$crap = exec('ls '.join(' ', $search_glob), $output, $status);
+					// now get all that are NOT in de DB
+					$q = "INSERT INTO temp_files (folder, filename) VALUES ";
+					$t_q = '';
+					foreach ($output as $output_file) {
+						// split the ouput into folder and file
+						// eg ../admin/test.php is ../admin/ and test.php
+						preg_match("/([\.\/\w]+\/)+(\w+\.\w{1,})$/", $output_file, $matches);
+						// if named config.php, skip
+						if ($matches[2] != 'config.php') {
+							if ($t_q) {
+								$t_q .= ', ';
+							}
+							$t_q .= "('".$form->dbEscapeString($matches[1])."', '".$form->dbEscapeString($matches[2])."')";
+						}
+					}
+					$form->dbExec($q.$t_q, 'NULL');
 					$elements[] = $form->formCreateElement('filename');
 				} else {
 					// show file menu
@@ -372,6 +412,7 @@ if ($form->my_page_name == 'edit_order') {
 					$DATA['filename_exist'] = 1;
 					$DATA['filename'] = $form->table_array['filename']['value'];
 				} // File Name View IF
+				$elements[] = $form->formCreateElement('hostname');
 				$elements[] = $form->formCreateElement('name');
 				// $elements[] = $form->formCreateElement('tag');
 				// $elements[] = $form->formCreateElement('min_acl');

www/includes/table_arrays/array_edit_pages.php

@@ -12,11 +12,16 @@ $edit_pages = array(
 			'output_name' => 'Add File ...',
 			'mandatory' => 1,
 			'type' => 'drop_down_db',
-			'query' => "SELECT DISTINCT temp_files.filename AS id, temp_files.filename AS name ".
+			'query' => "SELECT DISTINCT temp_files.filename AS id, temp_files.folder || temp_files.filename AS name ".
 				"FROM temp_files ".
 				"LEFT JOIN edit_page ep ON temp_files.filename = ep.filename ".
 				"WHERE ep.filename IS NULL"
 		),
+		'hostname' => array(
+			'value' => isset($GLOBALS['hostname']) ? $GLOBALS['hostname'] : '',
+			'output_name' => 'Hostname or folder',
+			'type' => 'text'
+		),
 		'name' => array(
 			'value' => isset($GLOBALS['name']) ? $GLOBALS['name'] : '',
 			'output_name' => 'Page name',
@@ -107,7 +112,7 @@ $edit_pages = array(
 				// "ORDER BY order_number"
 		)
 	),
-	'load_query' => "SELECT edit_page_id, filename, name, online, menu, popup FROM edit_page ORDER BY order_number",
+	'load_query' => "SELECT edit_page_id, CASE WHEN hostname IS NOT NULL THEN hostname ELSE ''::VARCHAR END || filename AS filename, name, online, menu, popup FROM edit_page ORDER BY order_number",
 	'table_name' => 'edit_page',
 	'show_fields' => array(
 		array(

www/lib/CoreLibs/ACL/Login.php

@@ -434,7 +434,9 @@ class Login extends \CoreLibs\DB\IO
 							$pages = array();
 							$pages_acl = array();
 							// set pages access
-							$q = "SELECT ep.edit_page_id, ep.cuid, epca.cuid AS content_alias_uid, ep.filename, ep.name AS edit_page_name, ep.order_number AS edit_page_order, ep.menu, ";
+							$q = "SELECT ep.edit_page_id, ep.cuid, epca.cuid AS content_alias_uid, ";
+							$q .= "ep.hostname, ep.filename, ep.name AS edit_page_name, ";
+							$q .= "ep.order_number AS edit_page_order, ep.menu, ";
 							$q .= "ep.popup, ep.popup_x, ep.popup_y, ep.online, ear.level, ear.type ";
 							$q .= "FROM edit_page ep ";
 							$q .= "LEFT JOIN edit_page epca ON (epca.edit_page_id = ep.content_alias_edit_page_id)";
@@ -450,6 +452,7 @@ class Login extends \CoreLibs\DB\IO
 									'edit_page_id' => $res['edit_page_id'],
 									'cuid' => $res['cuid'],
 									'content_alias_uid' => $res['content_alias_uid'], // for reference of content data on a differen page
+									'hostname' => $res['hostname'],
 									'filename' => $res['filename'],
 									'page_name' => $res['edit_page_name'],
 									'order' => $res['edit_page_order'],

www/lib/CoreLibs/Admin/Backend.php

@@ -287,7 +287,11 @@ class Backend extends \CoreLibs\DB\IO
 							}
 						}
 					}
-					$url = isset($data['filename']) ? $data['filename'] : '';
+					$url = '';
+					if (isset($data['hostname']) && $data['hostname']) {
+						$url .= $data['hostname'];
+					}
+					$url .= isset($data['filename']) ? $data['filename'] : '';
 					if (strlen($query_string)) {
 						$url .= '?'.$query_string;
 					}
@@ -295,7 +299,11 @@ class Backend extends \CoreLibs\DB\IO
 					// if page name matchs -> set selected flag
 					$selected = 0;
 					if (isset($data['filename']) &&
-						$this->getPageName() == $data['filename']
+						$this->getPageName() == $data['filename'] &&
+						(!isset($data['hostname']) || (
+							isset($data['hostname']) &&
+								(!$data['hostname'] || strstr($data['hostname'], CONTENT_PATH) !== false)
+						))
 					) {
 						$selected = 1;
 						$this->page_name = $name;
@@ -364,7 +372,7 @@ class Backend extends \CoreLibs\DB\IO
 	public function adbByteStringFormat($number): string
 	{
 		trigger_error('Method '.__METHOD__.' is deprecated', E_USER_DEPRECATED);
-		return $this->byteStringFormat($number);
+		return $this->humanReadableByteFormat($number);
 	}
 
 	/**

www/lib/CoreLibs/Basic.php

@@ -100,6 +100,10 @@ class Basic
 	// define check vars for the flags we can have
 	const CLASS_STRICT_MODE = 1;
 	const CLASS_OFF_COMPATIBLE_MODE = 2;
+	// define byteFormat
+	const BYTE_FORMAT_NOSPACE = 1;
+	const BYTE_FORMAT_ADJUST = 2;
+	const BYTE_FORMAT_SI = 4;
 	// control vars
 	/** @var bool compatible mode sets variable even if it is not defined */
 	private $set_compatible = true;
@@ -1136,6 +1140,8 @@ class Basic
 			return "href=##QUOT##$_1$_2$_3##QUOT##";
 		} elseif ($atag) {
 			return $atag.$_2.$_3;
+		} else {
+			return $href;
 		}
 	}
 
@@ -1592,20 +1598,117 @@ class Basic
 	}
 
 	/**
+	 * WRAPPER call to new humanReadableByteFormat
 	 * converts bytes into formated string with KB, MB, etc
-	 * @param  string|int|float $number bytes as string int or pure int
-	 * @param  bool             $space  true (default) to add space between number and suffix
+	 * @param  string|int|float $bytes  bytes as string int or pure int
+	 * @param  bool             $space  default true, to add space between number and suffix
+	 * @param  bool             $adjust default false, always print two decimals (sprintf)
+	 * @param  bool             $si     default false, if set to true, use 1000 for calculation
 	 * @return string                   converted byte number (float) with suffix
+	 * @deprecated Use humanReadableByteFormat instead
 	 */
-	public static function byteStringFormat($number, bool $space = true): string
+	public static function byteStringFormat($bytes, bool $space = true, bool $adjust = false, bool $si = false): string
 	{
-		if (is_numeric($number) && $number > 0) {
-			// labels in order of size
-			$labels = array('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB');
-			// calc file size, round down too two digits, add label based max change
-			return round((float)$number / pow(1024, ($i = floor(log((float)$number, 1024)))), 2).($space ? ' ' : '').(isset($labels[(int)$i]) ? $labels[(int)$i] : '>EB');
+		// trigger_error('Method '.__METHOD__.' is deprecated, use humanReadableByteFormat', E_USER_DEPRECATED);
+		$flags = 0;
+		// match over the true/false flags to the new int style flag
+		// if space need to set 1
+		if ($space === false) {
+			$flags |= self::BYTE_FORMAT_NOSPACE;
+		}
+		// if adjust need to set 2
+		if ($adjust === true) {
+			$flags |= self::BYTE_FORMAT_ADJUST;
+		}
+		// if si need to set 3
+		if ($si === true) {
+			$flags |= self::BYTE_FORMAT_SI;
+		}
+
+		// call
+		return self::humanReadableByteFormat($bytes, $flags);
+	}
+
+
+	/**
+	 * This function replaces the old byteStringFormat
+	 *
+	 * Converts any number string to human readable byte format
+	 * Maxium is Exobytes and above that the Exobytes suffix is used for all
+	 * If more are needed only the correct short name for the suffix has to be
+	 * added to the labels array
+	 * On no number string it returns string as is
+	 * Source Idea: SOURCE: https://programming.guide/worlds-most-copied-so-snippet.html
+	 *
+	 * The class itself hast the following defined
+	 * BYTE_FORMAT_NOSPACE [1] turn off spaces between number and extension
+	 * BYTE_FORMAT_ADJUST  [2] use sprintf to always print two decimals
+	 * BYTE_FORMAT_SI      [3] use si standard 1000 instead of bytes 1024
+	 * To use the constant from outside use $class::CONSTANT
+	 * @param  string|int|float $bytes bytes as string int or pure int
+	 * @param  int              $flags bitwise flag with use space turned on
+	 * @return string                  converted byte number (float) with suffix
+	 */
+	public static function humanReadableByteFormat($bytes, int $flags = 0): string
+	{
+		// if not numeric, return as is
+		if (is_numeric($bytes)) {
+			// flags bit wise check
+			// remove space between number and suffix
+			if ($flags & self::BYTE_FORMAT_NOSPACE) {
+				$space = false;
+			} else {
+				$space = true;
+			}
+			// use sprintf instead of round
+			if ($flags & self::BYTE_FORMAT_ADJUST) {
+				$adjust = true;
+			} else {
+				$adjust = false;
+			}
+			// use SI 1000 mod and not 1024 mod
+			if ($flags & self::BYTE_FORMAT_SI) {
+				$si = true;
+			} else {
+				$si = false;
+			}
+
+			// si or normal
+			$unit = $si ? 1000 : 1024;
+			// always positive
+			$abs_bytes = $bytes == PHP_INT_MIN ? PHP_INT_MAX : abs($bytes);
+			// smaller than unit is always B
+			if ($abs_bytes < $unit) {
+				return $bytes.'B';
+			}
+			// labels in order of size [Y, Z]
+			$labels = array('', 'K', 'M', 'G', 'T', 'P', 'E');
+			// exp position calculation
+			$exp = floor(log($abs_bytes, $unit));
+			// avoid printing out anything larger than max labels
+			if ($exp >= count($labels)) {
+				$exp = count($labels) - 1;
+			}
+			// deviation calculation
+			$dev = pow($unit, $exp) * ($unit - 0.05);
+			// shift the exp +1 for on the border units
+			if ($exp < 6 &&
+				$abs_bytes > ($dev - (((int)$dev & 0xfff) == 0xd00 ? 52 : 0))
+			) {
+				$exp ++;
+			}
+			// label name, including leading space if flagged
+			$pre = ($space ? ' ' : '').(isset($labels[$exp]) ? $labels[$exp] : '>E').($si ? 'i' : '').'B';
+			$bytes_calc = $abs_bytes / pow($unit, $exp);
+			if ($adjust) {
+				return sprintf("%.2f%sB", $bytes_calc, $pre);
+			} else {
+				return round($bytes_calc, 2).$pre;
+			}
+		} else {
+			// if anything other return as string
+			return (string)$bytes;
 		}
-		return (string)$number;
 	}
 
 	/**

www/lib/CoreLibs/DB/IO.php

@@ -321,7 +321,7 @@ class IO extends \CoreLibs\Basic
 		$this->db_user = $db_config['db_user'] ?? '';
 		$this->db_pwd = $db_config['db_pass'] ?? '';
 		$this->db_host = $db_config['db_host'] ?? '';
-		$this->db_port = !empty($db_config['db_port']) ? $db_config['db_port'] : '5432';
+		$this->db_port = !empty($db_config['db_port']) ? $db_config['db_port'] : 5432;
 		$this->db_schema = !empty($db_config['db_schema']) ? $db_config['db_schema'] : ''; // do not set to 'public' if not set, because the default is already public
 		$this->db_encoding = !empty($db_config['db_encoding']) ? $db_config['db_encoding'] : '';
 		$this->db_type = $db_config['db_type'] ?? '';

www/lib/CoreLibs/Output/Form/Generate.php

@@ -1974,7 +1974,9 @@ class Generate extends \CoreLibs\DB\Extended\ArrayIO
 		} else {
 			// only create query if we have a primary key
 			// reads directly from the reference table
-			if (isset($this->table_array[$this->int_pk_name]['value'])) {
+			if (isset($this->table_array[$this->int_pk_name]['value']) &&
+				$this->table_array[$this->int_pk_name]['value']
+			) {
 				$q = 'SELECT '.implode(', ', $q_select).' FROM '.$table_name.' WHERE '.$this->int_pk_name.' = '.$this->table_array[$this->int_pk_name]['value'];
 			}
 		}

www/lib/CoreLibs/Template/SmartyExtend.php

@@ -81,6 +81,7 @@ class SmartyExtend extends SmartyBC
 	public $INCLUDES;
 	public $JAVASCRIPT;
 	public $CSS;
+	public $FONT;
 	public $PICTURES;
 	public $CACHE_PICTURES;
 	public $CACHE_PICTURES_ROOT;
@@ -175,6 +176,7 @@ class SmartyExtend extends SmartyBC
 		$this->setTemplateDir($this->TEMPLATE_PATH);
 		$this->JAVASCRIPT = LAYOUT.JS;
 		$this->CSS = LAYOUT.CSS;
+		$this->FONT = LAYOUT.FONT;
 		$this->PICTURES = LAYOUT.IMAGES;
 		$this->CACHE_PICTURES = LAYOUT.CACHE;
 		$this->CACHE_PICTURES_ROOT = ROOT.$this->CACHE_PICTURES;
@@ -334,6 +336,7 @@ class SmartyExtend extends SmartyBC
 		// default CMS settings
 		// define all needed smarty stuff for the general HTML/page building
 		$this->HEADER['CSS'] = CSS;
+		$this->HEADER['FONT'] = FONT;
 		$this->HEADER['JS'] = JS;
 		$this->HEADER['ENCODING'] = $this->encoding;
 		$this->HEADER['DEFAULT_ENCODING'] = DEFAULT_ENCODING;

www/lib/font

@@ -0,0 +1 @@
+Fonts/