Add base crc64 for small files
This commit is contained in:
cliphouse committed
1 parent
b759ba70f3
commit
6d0c83ab69
9 files changed
+257
No files matched your search
@@ -0,0 +1,73 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace tests;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/**
|
||||
* Test class for Create\Crc64
|
||||
* @coversDefaultClass \CoreLibs\Create\Crc64
|
||||
* @testdox \CoreLibs\Create\Crc64 method tests
|
||||
*/
|
||||
final class CoreLibsCreateCrc64Test extends TestCase
|
||||
{
|
||||
public function providerFiles(): array
|
||||
{
|
||||
return [
|
||||
'10kb' => [
|
||||
'file' => __DIR__ . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR . 'random_10KB.bin',
|
||||
'crc64_base64' => 'oMgkUkGrDME=',
|
||||
'crc64_hex' => 'a0c8245241ab0cc1',
|
||||
],
|
||||
'100kb' => [
|
||||
'file' => __DIR__ . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR . 'random_100KB.bin',
|
||||
'crc64_base64' => 'i9Lpf3Y3D3M=',
|
||||
'crc64_hex' => '8bd2e97f76370f73',
|
||||
],
|
||||
'1mb' => [
|
||||
'file' => __DIR__ . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR . 'random_1MB.bin',
|
||||
'crc64_base64' => 'WqYAirRJ0q8=',
|
||||
'crc64_hex' => '5aa6008ab449d2af',
|
||||
],
|
||||
'5mb' => [
|
||||
'file' => __DIR__ . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR . 'random_5MB.bin',
|
||||
'crc64_base64' => '9hS5kUxbgR0=',
|
||||
'crc64_hex' => 'f614b9914c5b811d',
|
||||
],
|
||||
'10mb' => [
|
||||
'file' => __DIR__ . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR . 'random_10MB.bin',
|
||||
'crc64_base64' => 'xDmtkk1JhjM=',
|
||||
'crc64_hex' => 'c439ad924d498633',
|
||||
],
|
||||
// everything above 10MB is too slow, sow do not check this
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @covers ::crc64
|
||||
* @covers ::crc64hex
|
||||
* @dataProvider providerFiles
|
||||
* @testdox generate crc64 for $file and expecting $crc64_base64 and $crc64_hex [$_dataName]
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function testCrc64(string $file, string $crc64_base64, string $crc64_hex): void
|
||||
{
|
||||
$this->assertSame(
|
||||
$crc64_base64,
|
||||
\CoreLibs\Create\Crc64::crc64($file),
|
||||
"Failed asserting base64 CRC64 for $file"
|
||||
);
|
||||
$this->assertSame(
|
||||
$crc64_hex,
|
||||
\CoreLibs\Create\Crc64::crc64hex($file),
|
||||
"Failed asserting hex CRC64 for $file"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// __END__
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,77 @@
|
||||
<?php // phpcs:ignore warning
|
||||
|
||||
/**
|
||||
* @phan-file-suppress PhanTypeSuspiciousStringExpression
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
error_reporting(E_ALL | E_ERROR | E_WARNING | E_PARSE | E_COMPILE_ERROR);
|
||||
|
||||
ob_start();
|
||||
|
||||
// basic class test file
|
||||
define('USE_DATABASE', false);
|
||||
// sample config
|
||||
require 'config.php';
|
||||
// define log file id
|
||||
$LOG_FILE_ID = 'classTest-crc64';
|
||||
ob_end_flush();
|
||||
|
||||
use CoreLibs\Create\Crc64;
|
||||
use CoreLibs\Debug\RunningTime;
|
||||
|
||||
$log = new CoreLibs\Logging\Logging([
|
||||
'log_folder' => BASE . LOG,
|
||||
'log_file_id' => $LOG_FILE_ID,
|
||||
'log_per_date' => true,
|
||||
]);
|
||||
|
||||
$PAGE_NAME = 'TEST CLASS: CRC64';
|
||||
print "<!DOCTYPE html>";
|
||||
print "<html><head><title>" . $PAGE_NAME . "</title></head>";
|
||||
print "<body>";
|
||||
print '<div><a href="class_test.php">Class Test Master</a></div>';
|
||||
print '<div><h1>' . $PAGE_NAME . '</h1></div>';
|
||||
|
||||
|
||||
$file = getcwd() . DIRECTORY_SEPARATOR . 'class_test.file.php';
|
||||
print "CRC64: $file: " . Crc64::crc64($file) . "<br>";
|
||||
print "CRC64HEX: $file: " . Crc64::crc64hex($file) . "<br>";
|
||||
|
||||
// big file check including speed
|
||||
$base_path = __DIR__ . DIRECTORY_SEPARATOR
|
||||
. '..' . DIRECTORY_SEPARATOR
|
||||
. '..' . DIRECTORY_SEPARATOR
|
||||
. '4dev' . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR .
|
||||
'Create' . DIRECTORY_SEPARATOR . 'files' . DIRECTORY_SEPARATOR;
|
||||
$max_size = 1024 * 1024 * 10;
|
||||
# open the base path and loop over each file in there
|
||||
$files = glob($base_path . '*');
|
||||
if (!empty($files)) {
|
||||
foreach ($files as $file) {
|
||||
if (!is_file($file)) {
|
||||
continue;
|
||||
}
|
||||
if (filesize($file) > $max_size) {
|
||||
print "Skipping large file: $file<br>";
|
||||
continue;
|
||||
}
|
||||
RunningTime::runningTime();
|
||||
print "CRC64: $file: " . Crc64::crc64($file) . "<br>";
|
||||
RunningTime::runningTime();
|
||||
print "Running Time: $file: " . RunningTime::runningTimeString() . "<br>";
|
||||
print "Running Time: $file: " . RunningTime::hrRunningTime() . "<br>";
|
||||
print "CRC64HEX: $file: " . Crc64::crc64hex($file) . "<br>";
|
||||
RunningTime::runningTime();
|
||||
print "Running Time: $file: " . RunningTime::runningTimeString() . "<br>";
|
||||
print "Running Time: $file: " . RunningTime::hrRunningTime() . "<br>";
|
||||
RunningTime::hrRunningTimeReset();
|
||||
}
|
||||
} else {
|
||||
print "No files found in base path: $base_path<br>";
|
||||
}
|
||||
|
||||
print "</body></html>";
|
||||
|
||||
// __END__
|
||||
@@ -120,6 +120,7 @@ $test_files = [
|
||||
'class_test.datetime.php' => 'Class Test: DATE/TIME',
|
||||
'class_test.array.php' => 'Class Test: ARRAY HANDLER',
|
||||
'class_test.file.php' => 'Class Test: FILE',
|
||||
'class_test.crc64.php' => 'Class Test: CRC64',
|
||||
'class_test.randomkey.php' => 'Class Test: RANDOM KEY',
|
||||
'class_test.system.php' => 'Class Test: SYSTEM',
|
||||
'class_test.readenvfile.php' => 'Class Test: READ ENV FILE',
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* AUTHOR: Clemens Schwaighofer
|
||||
* CREATED: 2026/10/15
|
||||
* DESCRIPTION:
|
||||
* AWS CRC64 Implementation
|
||||
* This can be replaced by:
|
||||
* https://github.com/awesomized/crc-fast-php-ext
|
||||
* This this needs: https://github.com/awesomized/crc-fast-rust
|
||||
* installed, and there is no easy binary package available.
|
||||
*
|
||||
* NOTE: for anything larger than 10MB this is absolutely NOT recommended
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace CoreLibs\Create;
|
||||
|
||||
class Crc64
|
||||
{
|
||||
/**
|
||||
* Internal CRC64 calculation for a given data string.
|
||||
*
|
||||
* @param string $data
|
||||
* @param int $crc
|
||||
* @return int
|
||||
*/
|
||||
private function crc64Calc(string $data, int $crc = 0): int
|
||||
{
|
||||
static $table = null;
|
||||
if ($table === null) {
|
||||
// reflected poly 0x9A6C9329AC4BC9B5 (built from halves, too big for a PHP int literal)
|
||||
$poly = (0x9A6C9329 << 32) | 0xAC4BC9B5;
|
||||
$table = [];
|
||||
for ($i = 0; $i < 256; $i++) {
|
||||
$c = $i;
|
||||
for ($k = 0; $k < 8; $k++) {
|
||||
// logical right shift: PHP >> is arithmetic, so mask off the sign bit
|
||||
$c = ($c & 1)
|
||||
? ((($c >> 1) & PHP_INT_MAX) ^ $poly)
|
||||
: (($c >> 1) & PHP_INT_MAX);
|
||||
}
|
||||
$table[$i] = $c;
|
||||
}
|
||||
}
|
||||
$crc = ~$crc; // xor 0xFFFFFFFFFFFFFFFF
|
||||
foreach (unpack('C*', $data) ?: [] as $b) {
|
||||
$crc = $table[($crc ^ $b) & 0xFF] ^ (($crc >> 8) & 0x00FFFFFFFFFFFFFF);
|
||||
}
|
||||
return ~$crc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns int value of the CRC64 checksum for the given file.
|
||||
*
|
||||
* @param string $path
|
||||
* @param int<1,16777216> $chunkSize
|
||||
* @return int
|
||||
*/
|
||||
private function crc64CalcFile(string $path, int $chunkSize = 8 * 1024 * 1024): int
|
||||
{
|
||||
$fh = fopen($path, 'rb');
|
||||
if ($fh === false) {
|
||||
throw new \RuntimeException("Cannot open $path");
|
||||
}
|
||||
$crc = 0;
|
||||
try {
|
||||
while (!feof($fh)) {
|
||||
$chunk = fread($fh, $chunkSize);
|
||||
if ($chunk === false) {
|
||||
throw new \RuntimeException("Read error on $path");
|
||||
}
|
||||
$crc = $this->crc64Calc($chunk, $crc);
|
||||
}
|
||||
} finally {
|
||||
fclose($fh);
|
||||
}
|
||||
return $crc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Base64-encoded CRC64 checksum for the given file.
|
||||
*
|
||||
* @param string $path
|
||||
* @return string
|
||||
*/
|
||||
public static function crc64(string $path): string
|
||||
{
|
||||
return base64_encode(pack('J', (new self())->crc64CalcFile($path))); // J = uint64 big-endian
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the 16-character lowercase hexadecimal CRC64 checksum for the given file.
|
||||
* Same as base64.b64decode(...).hex()
|
||||
*
|
||||
* @param string $path
|
||||
* @return string
|
||||
*/
|
||||
public static function crc64hex(string $path): string
|
||||
{
|
||||
return sprintf('%016x', (new self())->crc64CalcFile($path));
|
||||
}
|
||||
}
|
||||
|
||||
// __END__
|
||||
Reference in new issue
Block a user