From 2d30d1d160c61ba7b7aad6506e9942197a6b9439 Mon Sep 17 00:00:00 2001 From: Clemens Schwaighofer Date: Mon, 7 Apr 2025 17:27:13 +0900 Subject: [PATCH] Rewrite DB param lookup * Correct wrong comment lookup * simplify regex by excluding comment and string blocks before * simpler lookup for each type * update checks for more tests for various special cases In DB IO * add a function to return all placeholders found in a query * only numbered parameters are looked up --- 4dev/tests/DB/CoreLibsDBIOTest.php | 47 ++-- .../class_test.db.convert-placeholder.php | 55 ++++- www/admin/class_test.db.encryption.php | 2 +- www/admin/class_test.db.query-placeholder.php | 4 +- www/lib/CoreLibs/DB/IO.php | 11 + www/lib/CoreLibs/DB/SQL/PgSQL.php | 21 +- .../DB/Support/ConvertPlaceholder.php | 201 +++++++++--------- 7 files changed, 208 insertions(+), 133 deletions(-) diff --git a/4dev/tests/DB/CoreLibsDBIOTest.php b/4dev/tests/DB/CoreLibsDBIOTest.php index d85de03f..d4e62d90 100644 --- a/4dev/tests/DB/CoreLibsDBIOTest.php +++ b/4dev/tests/DB/CoreLibsDBIOTest.php @@ -4744,7 +4744,7 @@ final class CoreLibsDBIOTest extends TestCase $res = $db->dbReturnRowParams($query_select, ['CONVERT_TYPE_TEST']); // all hast to be string foreach ($res as $key => $value) { - $this->assertIsString($value, 'Aseert string for column: ' . $key); + $this->assertIsString($value, 'Assert string for column: ' . $key); } // convert base only $db->dbSetConvertFlag(Convert::on); @@ -4757,10 +4757,10 @@ final class CoreLibsDBIOTest extends TestCase } switch ($type_layout[$name]) { case 'int': - $this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name); + $this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name); break; default: - $this->assertIsString($value, 'Aseert string for column: ' . $key . '/' . $name); + $this->assertIsString($value, 'Assert string for column: ' . $key . '/' . $name); break; } } @@ -4774,13 +4774,13 @@ final class CoreLibsDBIOTest extends TestCase } switch ($type_layout[$name]) { case 'int': - $this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name); + $this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name); break; case 'float': - $this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name); + $this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name); break; default: - $this->assertIsString($value, 'Aseert string for column: ' . $key . '/' . $name); + $this->assertIsString($value, 'Assert string for column: ' . $key . '/' . $name); break; } } @@ -4794,17 +4794,17 @@ final class CoreLibsDBIOTest extends TestCase } switch ($type_layout[$name]) { case 'int': - $this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name); + $this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name); break; case 'float': - $this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name); + $this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name); break; case 'json': case 'jsonb': - $this->assertIsArray($value, 'Aseert array for column: ' . $key . '/' . $name); + $this->assertIsArray($value, 'Assert array for column: ' . $key . '/' . $name); break; default: - $this->assertIsString($value, 'Aseert string for column: ' . $key . '/' . $name); + $this->assertIsString($value, 'Assert string for column: ' . $key . '/' . $name); break; } } @@ -4818,25 +4818,25 @@ final class CoreLibsDBIOTest extends TestCase } switch ($type_layout[$name]) { case 'int': - $this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name); + $this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name); break; case 'float': - $this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name); + $this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name); break; case 'json': case 'jsonb': - $this->assertIsArray($value, 'Aseert array for column: ' . $key . '/' . $name); + $this->assertIsArray($value, 'Assert array for column: ' . $key . '/' . $name); break; case 'bytea': // for hex types it must not start with \x $this->assertStringStartsNotWith( '\x', $value, - 'Aseert bytes not starts with \x for column: ' . $key . '/' . $name + 'Assert bytes not starts with \x for column: ' . $key . '/' . $name ); break; default: - $this->assertIsString($value, 'Aseert string for column: ' . $key . '/' . $name); + $this->assertIsString($value, 'Assert string for column: ' . $key . '/' . $name); break; } } @@ -5235,6 +5235,9 @@ final class CoreLibsDBIOTest extends TestCase $3 -- comment 3 , $4 + -- ignore $5, $6 + -- $7, $8 + -- digest($9, 10) ) SQL, 'count' => 4, @@ -5305,6 +5308,20 @@ final class CoreLibsDBIOTest extends TestCase SQL, 'count' => 2, 'convert' => false, + ], + // a text string with escaped quite + 'text string, with escaped quote' => [ + 'query' => << 2, + 'convert' => false, ] ]; } diff --git a/www/admin/class_test.db.convert-placeholder.php b/www/admin/class_test.db.convert-placeholder.php index 6504326f..8257805c 100644 --- a/www/admin/class_test.db.convert-placeholder.php +++ b/www/admin/class_test.db.convert-placeholder.php @@ -21,6 +21,7 @@ ob_end_flush(); use CoreLibs\Debug\Support; use CoreLibs\DB\Support\ConvertPlaceholder; +use CoreLibs\Convert\Html; $log = new CoreLibs\Logging\Logging([ 'log_folder' => BASE . LOG, @@ -38,10 +39,12 @@ print '

' . $PAGE_NAME . '

'; print "LOGFILE NAME: " . $log->getLogFile() . "
"; print "LOGFILE ID: " . $log->getLogFileId() . "
"; -print "Lookup Regex:
" . ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS . "
"; -print "Replace Named Regex:
" . ConvertPlaceholder::REGEX_REPLACE_NAMED . "
"; -print "Replace Named Regex:
" . ConvertPlaceholder::REGEX_REPLACE_QUESTION_MARK . "
"; -print "Replace Named Regex:
" . ConvertPlaceholder::REGEX_REPLACE_NUMBERED . "
"; +print "Lookup Regex:
" . Html::htmlent(ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS) . "
"; +print "Lookup Numbered Regex:
" . Html::htmlent(ConvertPlaceholder::REGEX_LOOKUP_NUMBERED) . "
"; +print "Replace Named Regex:
" . Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_NAMED) . "
"; +print "Replace Question Mark Regex:
"
+	. Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_QUESTION_MARK) . "
"; +print "Replace Numbered Regex:
" . Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_NUMBERED) . "
"; $uniqid = \CoreLibs\Create\Uids::uniqIdShort(); // $binary_data = $db->dbEscapeBytea(file_get_contents('class_test.db.php') ?: ''); @@ -91,40 +94,63 @@ RETURNING some_binary SQL; -print "[ALL] Convert: " +print "[ALL] Convert: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) . "
"; echo "
"; $query = "SELECT foo FROM bar WHERE baz = :baz AND buz = :baz AND biz = :biz AND boz = :bez"; $params = [':baz' => 'SETBAZ', ':bez' => 'SETBEZ', ':biz' => 'SETBIZ']; -print "[NO PARAMS] Convert: " +print "[NO PARAMS] Convert: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) . "
"; echo "
"; $query = "SELECT foo FROM bar WHERE baz = :baz AND buz = :baz AND biz = :biz AND boz = :bez"; $params = null; -print "[NO PARAMS] Convert: " +print "[NO PARAMS] Convert: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) . "
"; echo "
"; $query = "SELECT row_varchar FROM table_with_primary_key WHERE row_varchar <> :row_varchar"; $params = null; -print "[NO PARAMS] Convert: " +print "[NO PARAMS] Convert: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) . "
"; echo "
"; $query = "SELECT row_varchar, row_varchar_literal, row_int, row_date FROM table_with_primary_key"; $params = null; -print "[NO PARAMS] TEST: " +print "[NO PARAMS] TEST: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) . "
"; echo "
"; -print "[P-CONV]: " +$query = <<[All the same params] TEST: " + . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) + . "
"; +echo "
"; + +$query = << 1]; +print "[: param] TEST: " + . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params)) + . "
"; +echo "
"; + +print "[P-CONV]: " . Support::printAr( ConvertPlaceholder::updateParamList([ 'original' => [ @@ -186,6 +212,13 @@ SQL, 'params' => [\CoreLibs\Create\Uids::uniqIdShort(), 'string A-1', 1234], 'direction' => 'pg', ], + 'b?' => [ + 'query' => << [1234], + 'direction' => 'pg', + ], 'b:' => [ 'query' => << $data) { $query = $data['query']; $params = $data['params']; $direction = $data['direction']; - print "[$info] Convert: " + print "[$info] Convert: " . Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params, $direction)) . "
"; echo "
"; diff --git a/www/admin/class_test.db.encryption.php b/www/admin/class_test.db.encryption.php index 6d7701e2..7a0a827d 100644 --- a/www/admin/class_test.db.encryption.php +++ b/www/admin/class_test.db.encryption.php @@ -91,7 +91,7 @@ $db->dbExecParams( ] ); $cuuid = $db->dbGetReturningExt('cuuid'); -print "INSERTED: $cuuid
"; +print "INSERTED: " . print_r($cuuid, true) . "
"; print "LAST ERROR: " . $db->dbGetLastError(true) . "
"; // read back diff --git a/www/admin/class_test.db.query-placeholder.php b/www/admin/class_test.db.query-placeholder.php index c77cff6a..0516ed05 100644 --- a/www/admin/class_test.db.query-placeholder.php +++ b/www/admin/class_test.db.query-placeholder.php @@ -54,7 +54,7 @@ if (($dbh = $db->dbGetDbh()) instanceof \PgSql\Connection) { print "NO DB HANDLER
"; } // REGEX for placeholder count -print "Placeholder regex:
" . CoreLibs\DB\Support\ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS . "
"; +print "Placeholder lookup regex:
" . CoreLibs\DB\Support\ConvertPlaceholder::REGEX_LOOKUP_NUMBERED . "
"; // turn on debug replace for placeholders $db->dbSetDebugReplacePlaceholder(true); @@ -148,6 +148,7 @@ RETURNING bigint_a, number_real, number_double, numeric_3, uuid_var SQL; +print "Placeholders:
" . print_r($db->dbGetQueryParamPlaceholders($query_insert), true) . "
";
 $status = $db->dbExecParams($query_insert, $query_params);
 echo "*
"; echo "INSERT ALL COLUMN TYPES: " @@ -326,6 +327,7 @@ SQL, ) { print "RES: " . Support::prAr($res) . "
"; } +print "PL: " . Support::PrAr($db->dbGetPlaceholderConverted()) . "
"; print "ERROR: " . $db->dbGetLastError(true) . "
"; print ""; diff --git a/www/lib/CoreLibs/DB/IO.php b/www/lib/CoreLibs/DB/IO.php index a10a089b..9f60d38a 100644 --- a/www/lib/CoreLibs/DB/IO.php +++ b/www/lib/CoreLibs/DB/IO.php @@ -4283,6 +4283,17 @@ class IO return $this->field_names[$pos] ?? false; } + /** + * get all the $ placeholders + * + * @param string $query + * @return array + */ + public function dbGetQueryParamPlaceholders(string $query): array + { + return $this->db_functions->__dbGetQueryParams($query); + } + /** * Return a field type for a field name or pos, * will return false if field is not found in list diff --git a/www/lib/CoreLibs/DB/SQL/PgSQL.php b/www/lib/CoreLibs/DB/SQL/PgSQL.php index a343c8bc..352b6a5e 100644 --- a/www/lib/CoreLibs/DB/SQL/PgSQL.php +++ b/www/lib/CoreLibs/DB/SQL/PgSQL.php @@ -978,12 +978,12 @@ class PgSQL implements Interface\SqlFunctions } /** - * Count placeholder queries. $ only + * Get the all the $ params, unique list * * @param string $query - * @return int + * @return array */ - public function __dbCountQueryParams(string $query): int + public function __dbGetQueryParams(string $query): array { $matches = []; // regex for params: only stand alone $number allowed @@ -998,11 +998,22 @@ class PgSQL implements Interface\SqlFunctions // Matches in 1:, must be array_filtered to remove empty, count with array_unique // Regex located in the ConvertPlaceholder class preg_match_all( - ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS, + ConvertPlaceholder::REGEX_LOOKUP_NUMBERED, $query, $matches ); - return count(array_unique(array_filter($matches[3]))); + return array_unique(array_filter($matches[ConvertPlaceholder::MATCHING_POS])); + } + + /** + * Count placeholder queries. $ only + * + * @param string $query + * @return int + */ + public function __dbCountQueryParams(string $query): int + { + return count($this->__dbGetQueryParams($query)); } } diff --git a/www/lib/CoreLibs/DB/Support/ConvertPlaceholder.php b/www/lib/CoreLibs/DB/Support/ConvertPlaceholder.php index 484b7828..533b6a3a 100644 --- a/www/lib/CoreLibs/DB/Support/ConvertPlaceholder.php +++ b/www/lib/CoreLibs/DB/Support/ConvertPlaceholder.php @@ -14,76 +14,57 @@ namespace CoreLibs\DB\Support; class ConvertPlaceholder { - // NOTE for missing: range */+ are not iplemented in the regex below, but - is for now - // NOTE some combinations are allowed, but the query will fail before this - /** @var string split regex, entries before $ group */ - private const PATTERN_QUERY_SPLIT = - '\?\?|' // UNKNOWN: double ??, is this to avoid something? - . '[\(,]|' // for ',' and '(' mostly in INSERT or ANY() - . '[<>=]|' // general set for <, >, = in any query with any combination - . '\^@|' // text search for start from text with ^@ - . '\|\||' // concats two elements - . '&&|' // array overlap - . '\-\|\-|' // range overlap for array - . '[^-]-{1}|' // single -, used in JSON too - . '->|->>|#>|#>>|@>|<@|@@|@\?|\?{1}|\?\||\?&|#-|' // JSON searches, Array searchs, etc - . 'THEN|ELSE' // command parts (CASE) - ; - /** @var string the main regex including the pattern query split */ - private const PATTERN_ELEMENT = '(?:\'.*?\')?\s*(?:' . self::PATTERN_QUERY_SPLIT . ')\s*'; + /** @var string text block in SQL, single quited + * Note that does not include $$..$$ strings or anything with token name or nested ones + */ + private const PATTERN_TEXT_BLOCK_SINGLE_QUOTE = '(?:\'(?:[^\'\\\\]|\\\\.)*\')'; + /** @var string text block in SQL, dollar quoted + * NOTE: if this is added everything shifts by one lookup number + */ + private const PATTERN_TEXT_BLOCK_DOLLAR = '(?:\$([^$]*)\$.*?\$\1\$)'; /** @var string comment regex - * anything that starts with -- and ends with a line break but any character that is not line break inbetween */ - private const PATTERN_COMMENT = '(?:\-\-[^\r\n]*?\r?\n)*\s*'; - /** @var string parts to ignore in the SQL */ - private const PATTERN_IGNORE = - // digit -> ignore - '\d+|' - // other string -> ignore - . '(?:\'.*?\')|'; - /** @var string named parameters */ - private const PATTERN_NAMED = '(:\w+)'; - /** @var string question mark parameters */ - private const PATTERN_QUESTION_MARK = '(?:(?:\?\?)?\s*(\?{1}))'; - /** @var string numbered parameters */ + * anything that starts with -- and ends with a line break but any character that is not line break inbetween + * this is the FIRST thing in the line and will skip any further lookups */ + private const PATTERN_COMMENT = '(?:\-\-[^\r\n]*?\r?\n)'; + // below are the params lookups + /** @var string named parameters, must start with single : */ + private const PATTERN_NAMED = '((? add line break to matches in "." . '/s'; + /** @var string lookup for only numbered placeholders */ + public const REGEX_LOOKUP_NUMBERED = '/' + . self::PATTERN_COMMENT . '|' + . self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|' + . self::PATTERN_TEXT_BLOCK_DOLLAR . '|' + // match for replace part + . '(?:' + // $n numbered part (\PG php) [1] + . self::PATTERN_NUMBERED + // end match + . ')' + . '/s'; + /** @var int position for regex in full placeholder lookup: named */ + public const LOOOKUP_NAMED_POS = 2; + /** @var int position for regex in full placeholder lookup: question mark */ + public const LOOOKUP_QUESTION_MARK_POS = 3; + /** @var int position for regex in full placeholder lookup: numbered */ + public const LOOOKUP_NUMBERED_POS = 4; + /** @var int matches position for replacement and single lookup */ + public const MATCHING_POS = 2; /** * Convert PDO type query with placeholders to \PG style and vica versa @@ -132,11 +133,12 @@ class ConvertPlaceholder $found = -1; } /** @var array 1: named */ - $named_matches = array_filter($matches[1]); + $named_matches = array_filter($matches[self::LOOOKUP_NAMED_POS]); /** @var array 2: open ? */ - $qmark_matches = array_filter($matches[2]); + $qmark_matches = array_filter($matches[self::LOOOKUP_QUESTION_MARK_POS]); /** @var array 3: $n matches */ - $numbered_matches = array_filter($matches[3]); + $numbered_matches = array_filter($matches[self::LOOOKUP_NUMBERED_POS]); + // print "**MATCHES**:
" . print_r($matches, true) . "
"; // count matches $count_named = count(array_unique($named_matches)); $count_qmark = count($qmark_matches); @@ -235,38 +237,37 @@ class ConvertPlaceholder $empty_params = $converted_placeholders['original']['empty_params']; switch ($converted_placeholders['type']) { case 'named': - // 0: full - // 0: full - // 1: pre part - // 2: keep part UNLESS '3' is set - // 3: replace part :named + // 1: replace part :named $pos = 0; $query_new = preg_replace_callback( self::REGEX_REPLACE_NAMED, function ($matches) use (&$pos, &$params_new, &$params_lookup, $params, $empty_params) { - // only count up if $match[3] is not yet in lookup table - if (!empty($matches[3]) && empty($params_lookup[$matches[3]])) { + if (!isset($matches[self::MATCHING_POS])) { + throw new \RuntimeException( + 'Cannot lookup ' . self::MATCHING_POS . ' in matches list', + 209 + ); + } + $match = $matches[self::MATCHING_POS]; + // only count up if $match[1] is not yet in lookup table + if (empty($params_lookup[$match])) { $pos++; - $params_lookup[$matches[3]] = '$' . $pos; + $params_lookup[$match] = '$' . $pos; // skip params setup if param list is empty if (!$empty_params) { - $params_new[] = $params[$matches[3]] ?? + $params_new[] = $params[$match] ?? throw new \RuntimeException( - 'Cannot lookup ' . $matches[3] . ' in params list', + 'Cannot lookup ' . $match . ' in params list', 210 ); } } // add the connectors back (1), and the data sets only if no replacement will be done - return $matches[1] . ( - empty($matches[3]) ? - $matches[2] : - $params_lookup[$matches[3]] ?? - throw new \RuntimeException( - 'Cannot lookup ' . $matches[3] . ' in params lookup list', - 211 - ) - ); + return $params_lookup[$match] ?? + throw new \RuntimeException( + 'Cannot lookup ' . $match . ' in params lookup list', + 211 + ); }, $converted_placeholders['original']['query'] ); @@ -276,61 +277,61 @@ class ConvertPlaceholder // order and data stays the same $params_new = $params ?? []; } - // 0: full - // 1: pre part - // 2: keep part UNLESS '3' is set - // 3: replace part ? + // 1: replace part ? $pos = 0; $query_new = preg_replace_callback( self::REGEX_REPLACE_QUESTION_MARK, function ($matches) use (&$pos, &$params_lookup) { + if (!isset($matches[self::MATCHING_POS])) { + throw new \RuntimeException( + 'Cannot lookup ' . self::MATCHING_POS . ' in matches list', + 229 + ); + } + $match = $matches[self::MATCHING_POS]; // only count pos up for actual replacements we will do - if (!empty($matches[3])) { + if (!empty($match)) { $pos++; $params_lookup[] = '$' . $pos; } // add the connectors back (1), and the data sets only if no replacement will be done - return $matches[1] . ( - empty($matches[3]) ? - $matches[2] : - '$' . $pos - ); + return '$' . $pos; }, $converted_placeholders['original']['query'] ); break; case 'numbered': - // 0: full - // 1: pre part - // 2: keep part UNLESS '3' is set - // 3: replace part $numbered + // 1: replace part $numbered $pos = 0; $query_new = preg_replace_callback( self::REGEX_REPLACE_NUMBERED, function ($matches) use (&$pos, &$params_new, &$params_lookup, $params, $empty_params) { - // only count up if $match[3] is not yet in lookup table - if (!empty($matches[3]) && empty($params_lookup[$matches[3]])) { + if (!isset($matches[self::MATCHING_POS])) { + throw new \RuntimeException( + 'Cannot lookup ' . self::MATCHING_POS . ' in matches list', + 239 + ); + } + $match = $matches[self::MATCHING_POS]; + // only count up if $match[1] is not yet in lookup table + if (empty($params_lookup[$match])) { $pos++; - $params_lookup[$matches[3]] = ':' . $pos . '_named'; + $params_lookup[$match] = ':' . $pos . '_named'; // skip params setup if param list is empty if (!$empty_params) { $params_new[] = $params[($pos - 1)] ?? throw new \RuntimeException( 'Cannot lookup ' . ($pos - 1) . ' in params list', - 220 + 230 ); } } // add the connectors back (1), and the data sets only if no replacement will be done - return $matches[1] . ( - empty($matches[3]) ? - $matches[2] : - $params_lookup[$matches[3]] ?? - throw new \RuntimeException( - 'Cannot lookup ' . $matches[3] . ' in params lookup list', - 221 - ) - ); + return $params_lookup[$match] ?? + throw new \RuntimeException( + 'Cannot lookup ' . $match . ' in params lookup list', + 231 + ); }, $converted_placeholders['original']['query'] );