Update DB IO for query hash storage and parameter count

The parameter count methods in the PgSQL class have changed
- the function returns a unique list of $ parameters

The count is now done in the DB IO part where it counts over the unique array

Query hash is stored like the query for the current run one (reset on dbExec call).
The method to create the hash is renamed to dbBuildQueryHash instead of "Get".
The dbGetQueryHash function now just returns the last set query hash. There is a matching dbResetQueryHash for unsetting the query hash.

4dev/tests/DB/CoreLibsDBIOTest.php

@@ -135,6 +135,7 @@ final class CoreLibsDBIOTest extends TestCase
 		}
 		// check if they already exist, drop them
 		if ($db->dbShowTableMetaData('table_with_primary_key') !== false) {
+			$db->dbExec("CREATE EXTENSION IF NOT EXISTS pgcrypto");
 			$db->dbExec("DROP TABLE table_with_primary_key");
 			$db->dbExec("DROP TABLE table_without_primary_key");
 			$db->dbExec("DROP TABLE test_meta");
@@ -4744,7 +4745,7 @@ final class CoreLibsDBIOTest extends TestCase
 		$res = $db->dbReturnRowParams($query_select, ['CONVERT_TYPE_TEST']);
 		// all hast to be string
 		foreach ($res as $key => $value) {
-			$this->assertIsString($value, 'Aseert string for column: ' . $key);
+			$this->assertIsString($value, 'Assert string for column: ' . $key);
 		}
 		// convert base only
 		$db->dbSetConvertFlag(Convert::on);
@@ -4757,10 +4758,10 @@ final class CoreLibsDBIOTest extends TestCase
 			}
 			switch ($type_layout[$name]) {
 				case 'int':
-					$this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name);
+					$this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name);
 					break;
 				default:
-					$this->assertIsString($value, 'Aseert string for column: ' . $key  . '/' . $name);
+					$this->assertIsString($value, 'Assert string for column: ' . $key  . '/' . $name);
 					break;
 			}
 		}
@@ -4774,13 +4775,13 @@ final class CoreLibsDBIOTest extends TestCase
 			}
 			switch ($type_layout[$name]) {
 				case 'int':
-					$this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name);
+					$this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name);
 					break;
 				case 'float':
-					$this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name);
+					$this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name);
 					break;
 				default:
-					$this->assertIsString($value, 'Aseert string for column: ' . $key  . '/' . $name);
+					$this->assertIsString($value, 'Assert string for column: ' . $key  . '/' . $name);
 					break;
 			}
 		}
@@ -4794,17 +4795,17 @@ final class CoreLibsDBIOTest extends TestCase
 			}
 			switch ($type_layout[$name]) {
 				case 'int':
-					$this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name);
+					$this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name);
 					break;
 				case 'float':
-					$this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name);
+					$this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name);
 					break;
 				case 'json':
 				case 'jsonb':
-					$this->assertIsArray($value, 'Aseert array for column: ' . $key . '/' . $name);
+					$this->assertIsArray($value, 'Assert array for column: ' . $key . '/' . $name);
 					break;
 				default:
-					$this->assertIsString($value, 'Aseert string for column: ' . $key  . '/' . $name);
+					$this->assertIsString($value, 'Assert string for column: ' . $key  . '/' . $name);
 					break;
 			}
 		}
@@ -4818,25 +4819,25 @@ final class CoreLibsDBIOTest extends TestCase
 			}
 			switch ($type_layout[$name]) {
 				case 'int':
-					$this->assertIsInt($value, 'Aseert int for column: ' . $key . '/' . $name);
+					$this->assertIsInt($value, 'Assert int for column: ' . $key . '/' . $name);
 					break;
 				case 'float':
-					$this->assertIsFloat($value, 'Aseert float for column: ' . $key . '/' . $name);
+					$this->assertIsFloat($value, 'Assert float for column: ' . $key . '/' . $name);
 					break;
 				case 'json':
 				case 'jsonb':
-					$this->assertIsArray($value, 'Aseert array for column: ' . $key . '/' . $name);
+					$this->assertIsArray($value, 'Assert array for column: ' . $key . '/' . $name);
 					break;
 				case 'bytea':
 					// for hex types it must not start with \x
 					$this->assertStringStartsNotWith(
 						'\x',
 						$value,
-						'Aseert bytes not starts with \x for column: ' . $key . '/' . $name
+						'Assert bytes not starts with \x for column: ' . $key . '/' . $name
 					);
 					break;
 				default:
-					$this->assertIsString($value, 'Aseert string for column: ' . $key  . '/' . $name);
+					$this->assertIsString($value, 'Assert string for column: ' . $key  . '/' . $name);
 					break;
 			}
 		}
@@ -5008,8 +5009,8 @@ final class CoreLibsDBIOTest extends TestCase
 				)
 			),
 			($params === null ?
-				$db->dbGetQueryHash($query) :
+				$db->dbBuildQueryHash($query) :
-				$db->dbGetQueryHash($query, $params)
+				$db->dbBuildQueryHash($query, $params)
 			),
 			'Failed assertdbGetQueryHash '
 		);
@@ -5235,6 +5236,9 @@ final class CoreLibsDBIOTest extends TestCase
 					$3
 					-- comment 3
 					, $4
+					-- ignore $5, $6
+					-- $7, $8
+					-- digest($9, 10)
 				)
 				SQL,
 				'count' => 4,
@@ -5305,8 +5309,57 @@ final class CoreLibsDBIOTest extends TestCase
 				SQL,
 				'count' => 2,
 				'convert' => false,
+			],
+			// special $$ string case
+			'text string, with $ placehoders that could be seen as $$ string' => [
+				'query' => <<<SQL
+				SELECT row_int
+				FROM table_with_primary_key
+				WHERE
+					row_bytea = digest($3::VARCHAR, $4) OR
+					row_varchar = encode(digest($3, $4), 'hex') OR
+					row_bytea = hmac($3, $5, $4) OR
+					row_varchar = encode(hmac($3, $5, $4), 'hex') OR
+					row_bytea = pgp_sym_encrypt($3, $6) OR
+					row_varchar = encode(pgp_sym_encrypt($1, $6), 'hex') OR
+					row_varchar = CASE WHEN row_int = 1 THEN $1 ELSE $2 END
+				SQL,
+				'count' => 6,
+				'convert' => false,
+			],
+			// NOTE, in SQL heredoc we cannot write $$ strings parts
+			'text string, with $ placehoders are in $$ strings' => [
+				'query' => '
+				SELECT row_int
+				FROM table_with_primary_key
+				WHERE
+					row_varchar = $$some string$$ OR
+					row_varchar = $tag$some string$tag$ OR
+					row_varchar = $btag$some $1 string$btag$ OR
+					row_varchar = $btag$some $1 $subtag$ something $subtag$string$btag$ OR
+					row_varchar = $1
+				',
+				'count' => 1,
+				'convert' => false,
+			],
+			// a text string with escaped quite
+			'text string, with escaped quote' => [
+				'query' => <<<SQL
+				SELECT row_int
+				FROM table_with_primary_key
+				WHERE
+					row_varchar = 'foo bar bar baz $5' OR
+					row_varchar = 'foo bar '' barbar $6' OR
+					row_varchar = E'foo bar \' barbar $7' OR
+					row_varchar = CASE WHEN row_int = 1 THEN $1 ELSE $2 END
+				SQL,
+				'count' => 2,
+				'convert' => false,
 			]
 		];
+		$string = <<<SQL
+		'''
+		SQL;
 	}
 
 	/**

www/admin/class_test.db.convert-placeholder.php

@@ -21,6 +21,7 @@ ob_end_flush();
 
 use CoreLibs\Debug\Support;
 use CoreLibs\DB\Support\ConvertPlaceholder;
+use CoreLibs\Convert\Html;
 
 $log = new CoreLibs\Logging\Logging([
 	'log_folder' => BASE . LOG,
@@ -38,10 +39,12 @@ print '<div><h1>' . $PAGE_NAME . '</h1></div>';
 print "LOGFILE NAME: " . $log->getLogFile() . "<br>";
 print "LOGFILE ID: " . $log->getLogFileId() . "<br>";
 
-print "Lookup Regex: <pre>" . ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS . "</pre>";
+print "Lookup Regex: <pre>" . Html::htmlent(ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS) . "</pre>";
-print "Replace Named Regex: <pre>" . ConvertPlaceholder::REGEX_REPLACE_NAMED . "</pre>";
+print "Lookup Numbered Regex: <pre>" . Html::htmlent(ConvertPlaceholder::REGEX_LOOKUP_NUMBERED) . "</pre>";
-print "Replace Named Regex: <pre>" . ConvertPlaceholder::REGEX_REPLACE_QUESTION_MARK . "</pre>";
+print "Replace Named Regex: <pre>" . Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_NAMED) . "</pre>";
-print "Replace Named Regex: <pre>" . ConvertPlaceholder::REGEX_REPLACE_NUMBERED . "</pre>";
+print "Replace Question Mark Regex: <pre>"
+	. Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_QUESTION_MARK) . "</pre>";
+print "Replace Numbered Regex: <pre>" . Html::htmlent(ConvertPlaceholder::REGEX_REPLACE_NUMBERED) . "</pre>";
 
 $uniqid = \CoreLibs\Create\Uids::uniqIdShort();
 // $binary_data = $db->dbEscapeBytea(file_get_contents('class_test.db.php') ?: '');
@@ -91,40 +94,63 @@ RETURNING
 	some_binary
 SQL;
 
-print "[ALL] Convert: "
+print "<b>[ALL] Convert</b>: "
 	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
 	. "<br>";
 echo "<hr>";
 
 $query = "SELECT foo FROM bar WHERE baz = :baz AND buz = :baz AND biz = :biz AND boz = :bez";
 $params = [':baz' => 'SETBAZ', ':bez' => 'SETBEZ', ':biz' => 'SETBIZ'];
-print "[NO PARAMS] Convert: "
+print "<b>[NO PARAMS] Convert</b>: "
 	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
 	. "<br>";
 echo "<hr>";
 
 $query = "SELECT foo FROM bar WHERE baz = :baz AND buz = :baz AND biz = :biz AND boz = :bez";
 $params = null;
-print "[NO PARAMS] Convert: "
+print "<b>[NO PARAMS] Convert</b>: "
 	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
 	. "<br>";
 echo "<hr>";
 
 $query = "SELECT row_varchar FROM table_with_primary_key WHERE row_varchar <> :row_varchar";
 $params = null;
-print "[NO PARAMS] Convert: "
+print "<b>[NO PARAMS] Convert</b>: "
 	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
 	. "<br>";
 echo "<hr>";
 
 $query = "SELECT row_varchar, row_varchar_literal, row_int, row_date FROM table_with_primary_key";
 $params = null;
-print "[NO PARAMS] TEST: "
+print "<b>[NO PARAMS] TEST</b>: "
 	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
 	. "<br>";
 echo "<hr>";
 
-print "[P-CONV]: "
+$query = <<<SQL
+UPDATE table_with_primary_key SET
+	row_int = $1::INT, row_numeric = $1::NUMERIC, row_varchar = $1
+WHERE
+	row_varchar = $1
+SQL;
+$params = [1];
+print "<b>[All the same params] TEST</b>: "
+	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
+	. "<br>";
+echo "<hr>";
+
+$query = <<<SQL
+SELECT row_varchar, row_varchar_literal, row_int, row_date
+FROM table_with_primary_key
+WHERE row_varchar = :row_varchar
+SQL;
+$params = [':row_varchar' => 1];
+print "<b>[: param] TEST</b>: "
+	. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params))
+	. "<br>";
+echo "<hr>";
+
+print "<b>[P-CONV]</b>: "
 	. Support::printAr(
 		ConvertPlaceholder::updateParamList([
 			'original' => [
@@ -186,6 +212,13 @@ SQL,
 		'params' => [\CoreLibs\Create\Uids::uniqIdShort(), 'string A-1', 1234],
 		'direction' => 'pg',
 	],
+	'b?' => [
+		'query' => <<<SQL
+SELECT test FROM test_foo = ?
+SQL,
+		'params' => [1234],
+		'direction' => 'pg',
+	],
 	'b:' => [
 		'query' => <<<SQL
 INSERT INTO test_foo (
@@ -220,7 +253,7 @@ foreach ($test_queries as $info => $data) {
 	$query = $data['query'];
 	$params = $data['params'];
 	$direction = $data['direction'];
-	print "[$info] Convert: "
+	print "<b>[$info] Convert</b>: "
 		. Support::printAr(ConvertPlaceholder::convertPlaceholderInQuery($query, $params, $direction))
 		. "<br>";
 	echo "<hr>";

www/admin/class_test.db.encryption.php

@@ -42,7 +42,10 @@ print '<div><a href="class_test.php">Class Test Master</a></div>';
 print '<div><h1>' . $PAGE_NAME . '</h1></div>';
 
 // encryption key
-$key = CreateKey::generateRandomKey();
+$key_new = CreateKey::generateRandomKey();
+print "Secret Key NEW: " . $key_new . "<br>";
+// for reproducable test results
+$key = 'e475c19b9a3c8363feb06b51f5b73f1dc9b6f20757d4ab89509bf5cc70ed30ec';
 print "Secret Key: " . $key . "<br>";
 
 // test text
@@ -91,7 +94,7 @@ $db->dbExecParams(
 	]
 );
 $cuuid = $db->dbGetReturningExt('cuuid');
-print "INSERTED: $cuuid<br>";
+print "INSERTED: " . print_r($cuuid, true) . "<br>";
 print "LAST ERROR: " . $db->dbGetLastError(true) . "<br>";
 
 // read back
@@ -105,20 +108,35 @@ $res = $db->dbReturnRowParams(
 		-- in DB encryption
 		pg_digest_bytea, pg_digest_text,
 		pg_hmac_bytea, pg_hmac_text,
-		pg_crypt_bytea, pg_crypt_text
+		pg_crypt_bytea, pg_crypt_text,
+		encode(pg_crypt_bytea, 'hex') AS pg_crypt_bytea_hex,
+		pgp_sym_decrypt(pg_crypt_bytea, $2) AS from_pg_crypt_bytea,
+		pgp_sym_decrypt(decode(pg_crypt_text, 'hex'), $2) AS from_pg_crypt_text
 	FROM
 		test_encryption
 	WHERE
 		cuuid = $1
 	SQL,
 	[
-		$cuuid
+		$cuuid, $key
 	]
 );
 
 print "RES: <pre>" . Support::prAr($res) . "</pre><br>";
 
-// do compare
+if ($res === false) {
+	echo "Failed to run query<br>";
+} else {
+	if (hash_equals($string_hashed, $res['pg_digest_text'])) {
+		print "libsodium and pgcrypto hash match<br>";
+	}
+	if (hash_equals($string_hmac, $res['pg_hmac_text'])) {
+		print "libsodium and pgcrypto hash hmac match<br>";
+	}
+}
+
+
+// do compare for PHP and pgcrypto settings
 
 print "</body></html>";
 

www/admin/class_test.db.query-placeholder.php

@@ -54,7 +54,7 @@ if (($dbh = $db->dbGetDbh()) instanceof \PgSql\Connection) {
 	print "NO DB HANDLER<br>";
 }
 // REGEX for placeholder count
-print "Placeholder regex: <pre>" . CoreLibs\DB\Support\ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS . "</pre>";
+print "Placeholder lookup regex: <pre>" . CoreLibs\DB\Support\ConvertPlaceholder::REGEX_LOOKUP_NUMBERED . "</pre>";
 
 // turn on debug replace for placeholders
 $db->dbSetDebugReplacePlaceholder(true);
@@ -148,6 +148,7 @@ RETURNING
 	bigint_a, number_real, number_double, numeric_3,
 	uuid_var
 SQL;
+print "Placeholders: <pre>" . print_r($db->dbGetQueryParamPlaceholders($query_insert), true) . "<pre>";
 $status = $db->dbExecParams($query_insert, $query_params);
 echo "<b>*</b><br>";
 echo "INSERT ALL COLUMN TYPES: "
@@ -326,6 +327,7 @@ SQL,
 ) {
 	print "RES: " . Support::prAr($res) . "<br>";
 }
+print "PL: " . Support::PrAr($db->dbGetPlaceholderConverted()) . "<br>";
 print "ERROR: " . $db->dbGetLastError(true) . "<br>";
 
 print "</body></html>";

www/lib/CoreLibs/DB/IO.php

@@ -303,6 +303,8 @@ class IO
 	private string $query = '';
 	/** @var array<mixed> current params for query */
 	private array $params = [];
+	/** @var string current hash build from query and params */
+	private string $query_hash = '';
 	// if we do have a convert call, store the convert data in here, else it will be empty
 	/** @var array{}|array{original:array{query:string,params:array<mixed>},type:''|'named'|'numbered'|'question_mark',found:int,matches:array<string>,params_lookup:array<mixed>,query:string,params:array<mixed>} */
 	private array $placeholder_converted = [];
@@ -1319,7 +1321,7 @@ class IO
 	 */
 	private function __dbCountQueryParams(string $query): int
 	{
-		return $this->db_functions->__dbCountQueryParams($query);
+		return count($this->db_functions->__dbGetQueryParams($query));
 	}
 
 	/**
@@ -1382,6 +1384,8 @@ class IO
 		$this->query = $query;
 		// current params
 		$this->params = $params;
+		// empty on new
+		$this->query_hash = '';
 		// no query set
 		if (empty($this->query)) {
 			$this->__dbError(11);
@@ -1441,7 +1445,7 @@ class IO
 			$this->returning_id = true;
 		}
 		// import protection, hash needed
-		$query_hash = $this->dbGetQueryHash($this->query, $this->params);
+		$query_hash = $this->dbBuildQueryHash($this->query, $this->params);
 		// QUERY PARAMS: run query params check and rewrite
 		if ($this->dbGetConvertPlaceholder() === true) {
 			try {
@@ -1475,7 +1479,8 @@ class IO
 				return false;
 			}
 		}
-
+		// set query hash
+		$this->query_hash = $query_hash;
 		// $this->debug('DB IO', 'Q: ' . $this->query . ', RETURN: ' . $this->returning_id);
 		// for DEBUG, only on first time ;)
 		$this->__dbDebug(
@@ -1959,7 +1964,7 @@ class IO
 	{
 		// set start array
 		if ($query) {
-			$array = $this->cursor_ext[$this->dbGetQueryHash($query)] ?? [];
+			$array = $this->cursor_ext[$this->dbBuildQueryHash($query)] ?? [];
 		} else {
 			$array = $this->cursor_ext;
 		}
@@ -2361,7 +2366,7 @@ class IO
 			return false;
 		}
 		// create hash from query ...
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		// pre declare array
 		if (!isset($this->cursor_ext[$query_hash])) {
 			$this->cursor_ext[$query_hash] = [
@@ -2940,7 +2945,7 @@ class IO
 	public function dbCacheReset(string $query, array $params = []): bool
 	{
 		$this->__dbErrorReset();
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		// clears cache for this query
 		if (empty($this->cursor_ext[$query_hash]['query'])) {
 			$this->__dbWarning(18, context: [
@@ -2982,7 +2987,7 @@ class IO
 		if ($query === null) {
 			return $this->cursor_ext;
 		}
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		if (
 			!empty($this->cursor_ext) &&
 			isset($this->cursor_ext[$query_hash])
@@ -3012,7 +3017,7 @@ class IO
 			$this->__dbError(11);
 			return false;
 		}
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		if (
 			!empty($this->cursor_ext) &&
 			isset($this->cursor_ext[$query_hash])
@@ -3038,7 +3043,7 @@ class IO
 			$this->__dbError(11);
 			return false;
 		}
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		if (
 			!empty($this->cursor_ext) &&
 			isset($this->cursor_ext[$query_hash])
@@ -3064,7 +3069,7 @@ class IO
 	 */
 	public function dbResetQueryCalled(string $query, array $params = []): void
 	{
-		$this->query_called[$this->dbGetQueryHash($query, $params)] = 0;
+		$this->query_called[$this->dbBuildQueryHash($query, $params)] = 0;
 	}
 
 	/**
@@ -3077,7 +3082,7 @@ class IO
 	 */
 	public function dbGetQueryCalled(string $query, array $params = []): int
 	{
-		$query_hash = $this->dbGetQueryHash($query, $params);
+		$query_hash = $this->dbBuildQueryHash($query, $params);
 		if (!empty($this->query_called[$query_hash])) {
 			return $this->query_called[$query_hash];
 		} else {
@@ -4046,7 +4051,7 @@ class IO
 	}
 
 	/**
-	 * Returns hash for query
+	 * Creates hash for query and parameters
 	 * Hash is used in all internal storage systems for return data
 	 *
 	 * @param  string       $query  The query to create the hash from
@@ -4054,7 +4059,7 @@ class IO
 	 *                              data to create a unique call one, optional
 	 * @return string               Hash, as set by hash long
 	 */
-	public function dbGetQueryHash(string $query, array $params = []): string
+	public function dbBuildQueryHash(string $query, array $params = []): string
 	{
 		return Hash::hashLong(
 			$query . (
@@ -4104,6 +4109,26 @@ class IO
 		$this->params = [];
 	}
 
+	/**
+	 * get the current set query hash
+	 *
+	 * @return string Current Query hash
+	 */
+	public function dbGetQueryHash(): string
+	{
+		return $this->query_hash;
+	}
+
+	/**
+	 * reset query hash
+	 *
+	 * @return void
+	 */
+	public function dbResetQueryHash(): void
+	{
+		$this->query_hash = '';
+	}
+
 	/**
 	 * Returns the placeholder convert set or empty
 	 *
@@ -4283,6 +4308,17 @@ class IO
 		return $this->field_names[$pos] ?? false;
 	}
 
+	/**
+	 * get all the $ placeholders
+	 *
+	 * @param  string $query
+	 * @return array<string>
+	 */
+	public function dbGetQueryParamPlaceholders(string $query): array
+	{
+		return $this->db_functions->__dbGetQueryParams($query);
+	}
+
 	/**
 	 * Return a field type for a field name or pos,
 	 * will return false if field is not found in list

www/lib/CoreLibs/DB/SQL/Interface/SqlFunctions.php

@@ -379,9 +379,9 @@ interface SqlFunctions
 	 * Undocumented function
 	 *
 	 * @param  string $query
-	 * @return int
+	 * @return array<string>
 	 */
-	public function __dbCountQueryParams(string $query): int;
+	public function __dbGetQueryParams(string $query): array;
 }
 
 // __END__

www/lib/CoreLibs/DB/SQL/PgSQL.php

@@ -978,12 +978,12 @@ class PgSQL implements Interface\SqlFunctions
 	}
 
 	/**
-	 * Count placeholder queries. $ only
+	 * Get the all the $ params, as a unique list
 	 *
 	 * @param  string $query
-	 * @return int
+	 * @return array<string>
 	 */
-	public function __dbCountQueryParams(string $query): int
+	public function __dbGetQueryParams(string $query): array
 	{
 		$matches = [];
 		// regex for params: only stand alone $number allowed
@@ -998,11 +998,11 @@ class PgSQL implements Interface\SqlFunctions
 		// Matches in 1:, must be array_filtered to remove empty, count with array_unique
 		// Regex located in the ConvertPlaceholder class
 		preg_match_all(
-			ConvertPlaceholder::REGEX_LOOKUP_PLACEHOLDERS,
+			ConvertPlaceholder::REGEX_LOOKUP_NUMBERED,
 			$query,
 			$matches
 		);
-		return count(array_unique(array_filter($matches[3])));
+		return array_unique(array_filter($matches[ConvertPlaceholder::MATCHING_POS]));
 	}
 }
 

www/lib/CoreLibs/DB/Support/ConvertPlaceholder.php

@@ -14,76 +14,57 @@ namespace CoreLibs\DB\Support;
 
 class ConvertPlaceholder
 {
-	// NOTE for missing: range */+ are not iplemented in the regex below, but - is for now
+	/** @var string text block in SQL, single quited
-	// NOTE some combinations are allowed, but the query will fail before this
+	 * Note that does not include $$..$$ strings or anything with token name or nested ones
-	/** @var string split regex, entries before $ group */
+	*/
-	private const PATTERN_QUERY_SPLIT =
+	private const PATTERN_TEXT_BLOCK_SINGLE_QUOTE = '(?:\'(?:[^\'\\\\]|\\\\.)*\')';
-		'\?\?|' // UNKNOWN: double ??, is this to avoid something?
+	/** @var string text block in SQL, dollar quoted
-		. '[\(,]|' // for ',' and '(' mostly in INSERT or ANY()
+	 * NOTE: if this is added everything shifts by one lookup number
-		. '[<>=]|' // general set for <, >, = in any query with any combination
+	*/
-		. '\^@|' // text search for start from text with ^@
+	private const PATTERN_TEXT_BLOCK_DOLLAR = '(?:\$(\w*)\$.*?\$\1\$)';
-		. '\|\||' // concats two elements
-		. '&&|' // array overlap
-		. '\-\|\-|' // range overlap for array
-		. '[^-]-{1}|' // single -, used in JSON too
-		. '->|->>|#>|#>>|@>|<@|@@|@\?|\?{1}|\?\||\?&|#-|' // JSON searches, Array searchs, etc
-		. 'THEN|ELSE' // command parts (CASE)
-	;
-	/** @var string the main regex including  the pattern query split */
-	private const PATTERN_ELEMENT = '(?:\'.*?\')?\s*(?:' . self::PATTERN_QUERY_SPLIT . ')\s*';
 	/** @var string comment regex
-	 * anything that starts with -- and ends with a line break but any character that is not line break inbetween */
+	 * anything that starts with -- and ends with a line break but any character that is not line break inbetween
-	private const PATTERN_COMMENT = '(?:\-\-[^\r\n]*?\r?\n)*\s*';
+	 * this is the FIRST thing in the line and will skip any further lookups */
-	/** @var string parts to ignore in the SQL */
+	private const PATTERN_COMMENT = '(?:\-\-[^\r\n]*?\r?\n)';
-	private const PATTERN_IGNORE =
+	// below are the params lookups
-		// digit -> ignore
+	/** @var string named parameters, must start with single : */
-		'\d+|'
+	private const PATTERN_NAMED = '((?<!:):(?:\w+))';
-		// other string -> ignore
+	/** @var string question mark parameters, will catch any */
-		. '(?:\'.*?\')|';
+	private const PATTERN_QUESTION_MARK = '(\?{1})';
-	/** @var string named parameters */
+	/** @var string numbered parameters, can only start 1 to 9, second and further digits can be 0-9
-	private const PATTERN_NAMED = '(:\w+)';
+	 * This ignores the $$ ... $$ escape syntax. If we find something like this will fail
-	/** @var string question mark parameters */
+	 * It is recommended to use proper string escape quiting for writing data to the DB
-	private const PATTERN_QUESTION_MARK = '(?:(?:\?\?)?\s*(\?{1}))';
+	*/
-	/** @var string numbered parameters */
 	private const PATTERN_NUMBERED = '(\$[1-9]{1}(?:[0-9]{1,})?)';
 	// below here are full regex that will be used
 	/** @var string replace regex for named (:...) entries */
 	public const REGEX_REPLACE_NAMED = '/'
-		. '(' . self::PATTERN_ELEMENT . ')'
+		. self::PATTERN_COMMENT . '|'
-		. self::PATTERN_COMMENT
+		. self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|'
-		. '('
+		. self::PATTERN_TEXT_BLOCK_DOLLAR . '|'
-		. self::PATTERN_IGNORE
 		. self::PATTERN_NAMED
-		. ')'
 		. '/s';
 	/** @var string replace regex for question mark (?) entries */
 	public const REGEX_REPLACE_QUESTION_MARK = '/'
-		. '(' . self::PATTERN_ELEMENT . ')'
+		. self::PATTERN_COMMENT . '|'
-		. self::PATTERN_COMMENT
+		. self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|'
-		. '('
+		. self::PATTERN_TEXT_BLOCK_DOLLAR . '|'
-		. self::PATTERN_IGNORE
 		. self::PATTERN_QUESTION_MARK
-		. ')'
 		. '/s';
 	/** @var string replace regex for numbered ($n) entries */
 	public const REGEX_REPLACE_NUMBERED = '/'
-		. '(' . self::PATTERN_ELEMENT . ')'
+		. self::PATTERN_COMMENT . '|'
-		. self::PATTERN_COMMENT
+		. self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|'
-		. '('
+		. self::PATTERN_TEXT_BLOCK_DOLLAR . '|'
-		. self::PATTERN_IGNORE
 		. self::PATTERN_NUMBERED
-		. ')'
 		. '/s';
 	/** @var string the main lookup query for all placeholders */
 	public const REGEX_LOOKUP_PLACEHOLDERS = '/'
-		// prefix string part, must match towards
+		. self::PATTERN_COMMENT . '|'
-		// seperator for ( = , ? - [and json/jsonb in pg doc section 9.15]
+		. self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|'
-		. self::PATTERN_ELEMENT
+		. self::PATTERN_TEXT_BLOCK_DOLLAR . '|'
-		. self::PATTERN_COMMENT
 		// match for replace part
 		. '(?:'
-		// ignore parts
-		. self::PATTERN_IGNORE
 		// :name named part (PDO) [1]
 		. self::PATTERN_NAMED . '|'
 		// ? question mark part (PDO) [2]
@@ -94,6 +75,26 @@ class ConvertPlaceholder
 		. ')'
 		// single line -> add line break to matches in "."
 		. '/s';
+	/** @var string lookup for only numbered placeholders */
+	public const REGEX_LOOKUP_NUMBERED = '/'
+		. self::PATTERN_COMMENT . '|'
+		. self::PATTERN_TEXT_BLOCK_SINGLE_QUOTE . '|'
+		. self::PATTERN_TEXT_BLOCK_DOLLAR . '|'
+		// match for replace part
+		. '(?:'
+		// $n numbered part (\PG php) [1]
+		. self::PATTERN_NUMBERED
+		// end match
+		. ')'
+		. '/s';
+	/** @var int position for regex in full placeholder lookup: named */
+	public const LOOOKUP_NAMED_POS = 2;
+	/** @var int position for regex in full placeholder lookup: question mark */
+	public const LOOOKUP_QUESTION_MARK_POS = 3;
+	/** @var int position for regex in full placeholder lookup: numbered */
+	public const LOOOKUP_NUMBERED_POS = 4;
+	/** @var int matches position for replacement and single lookup */
+	public const MATCHING_POS = 2;
 
 	/**
 	 * Convert PDO type query with placeholders to \PG style and vica versa
@@ -132,11 +133,12 @@ class ConvertPlaceholder
 			$found = -1;
 		}
 		/** @var array<string> 1: named */
-		$named_matches = array_filter($matches[1]);
+		$named_matches = array_filter($matches[self::LOOOKUP_NAMED_POS]);
 		/** @var array<string> 2: open ? */
-		$qmark_matches = array_filter($matches[2]);
+		$qmark_matches = array_filter($matches[self::LOOOKUP_QUESTION_MARK_POS]);
 		/** @var array<string> 3: $n matches */
-		$numbered_matches = array_filter($matches[3]);
+		$numbered_matches = array_filter($matches[self::LOOOKUP_NUMBERED_POS]);
+		// print "**MATCHES**: <pre>" . print_r($matches, true) . "</pre>";
 		// count matches
 		$count_named = count(array_unique($named_matches));
 		$count_qmark = count($qmark_matches);
@@ -235,38 +237,37 @@ class ConvertPlaceholder
 		$empty_params = $converted_placeholders['original']['empty_params'];
 		switch ($converted_placeholders['type']) {
 			case 'named':
-				// 0: full
+				// 1: replace part :named
-				// 0: full
-				// 1: pre part
-				// 2: keep part UNLESS '3' is set
-				// 3: replace part :named
 				$pos = 0;
 				$query_new = preg_replace_callback(
 					self::REGEX_REPLACE_NAMED,
 					function ($matches) use (&$pos, &$params_new, &$params_lookup, $params, $empty_params) {
-						// only count up if $match[3] is not yet in lookup table
+						if (!isset($matches[self::MATCHING_POS])) {
-						if (!empty($matches[3]) && empty($params_lookup[$matches[3]])) {
+							throw new \RuntimeException(
+								'Cannot lookup ' . self::MATCHING_POS . ' in matches list',
+								209
+							);
+						}
+						$match = $matches[self::MATCHING_POS];
+						// only count up if $match[1] is not yet in lookup table
+						if (empty($params_lookup[$match])) {
 							$pos++;
-							$params_lookup[$matches[3]] = '$' . $pos;
+							$params_lookup[$match] = '$' . $pos;
 							// skip params setup if param list is empty
 							if (!$empty_params) {
-								$params_new[] = $params[$matches[3]] ??
+								$params_new[] = $params[$match] ??
 									throw new \RuntimeException(
-										'Cannot lookup ' . $matches[3] . ' in params list',
+										'Cannot lookup ' . $match . ' in params list',
 										210
 									);
 							}
 						}
 						// add the connectors back (1), and the data sets only if no replacement will be done
-						return $matches[1] . (
+						return $params_lookup[$match] ??
-							empty($matches[3]) ?
+							throw new \RuntimeException(
-								$matches[2] :
+								'Cannot lookup ' . $match . ' in params lookup list',
-								$params_lookup[$matches[3]] ??
+								211
-									throw new \RuntimeException(
+							);
-										'Cannot lookup ' . $matches[3] . ' in params lookup list',
-										211
-									)
-						);
 					},
 					$converted_placeholders['original']['query']
 				);
@@ -276,61 +277,61 @@ class ConvertPlaceholder
 					// order and data stays the same
 					$params_new = $params ?? [];
 				}
-				// 0: full
+				// 1: replace part ?
-				// 1: pre part
-				// 2: keep part UNLESS '3' is set
-				// 3: replace part ?
 				$pos = 0;
 				$query_new = preg_replace_callback(
 					self::REGEX_REPLACE_QUESTION_MARK,
 					function ($matches) use (&$pos, &$params_lookup) {
+						if (!isset($matches[self::MATCHING_POS])) {
+							throw new \RuntimeException(
+								'Cannot lookup ' . self::MATCHING_POS . ' in matches list',
+								229
+							);
+						}
+						$match = $matches[self::MATCHING_POS];
 						// only count pos up for actual replacements we will do
-						if (!empty($matches[3])) {
+						if (!empty($match)) {
 							$pos++;
 							$params_lookup[] = '$' . $pos;
 						}
 						// add the connectors back (1), and the data sets only if no replacement will be done
-						return $matches[1] . (
+						return '$' . $pos;
-							empty($matches[3]) ?
-								$matches[2] :
-								'$' . $pos
-						);
 					},
 					$converted_placeholders['original']['query']
 				);
 				break;
 			case 'numbered':
-				// 0: full
+				// 1: replace part $numbered
-				// 1: pre part
-				// 2: keep part UNLESS '3' is set
-				// 3: replace part $numbered
 				$pos = 0;
 				$query_new = preg_replace_callback(
 					self::REGEX_REPLACE_NUMBERED,
 					function ($matches) use (&$pos, &$params_new, &$params_lookup, $params, $empty_params) {
-						// only count up if $match[3] is not yet in lookup table
+						if (!isset($matches[self::MATCHING_POS])) {
-						if (!empty($matches[3]) && empty($params_lookup[$matches[3]])) {
+							throw new \RuntimeException(
+								'Cannot lookup ' . self::MATCHING_POS . ' in matches list',
+								239
+							);
+						}
+						$match = $matches[self::MATCHING_POS];
+						// only count up if $match[1] is not yet in lookup table
+						if (empty($params_lookup[$match])) {
 							$pos++;
-							$params_lookup[$matches[3]] = ':' . $pos . '_named';
+							$params_lookup[$match] = ':' . $pos . '_named';
 							// skip params setup if param list is empty
 							if (!$empty_params) {
 								$params_new[] = $params[($pos - 1)] ??
 									throw new \RuntimeException(
 										'Cannot lookup ' . ($pos - 1) . ' in params list',
-										220
+										230
 									);
 							}
 						}
 						// add the connectors back (1), and the data sets only if no replacement will be done
-						return $matches[1] . (
+						return $params_lookup[$match]  ??
-							empty($matches[3]) ?
+							throw new \RuntimeException(
-								$matches[2] :
+								'Cannot lookup ' . $match . ' in params lookup list',
-								$params_lookup[$matches[3]] ??
+								231
-								throw new \RuntimeException(
+							);
-									'Cannot lookup ' . $matches[3] . ' in params lookup list',
-									221
-								)
-						);
 					},
 					$converted_placeholders['original']['query']
 				);