vendor/blackbit/data-director/EventListener/AutomaticImportListener.php line 64

Open in your IDE?
  1. <?php
  2. /**
  3.  * Copyright Blackbit digital Commerce GmbH <info@blackbit.de>
  4.  *
  5.  * This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.
  6.  *
  7.  * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  8.  *
  9.  * You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  10.  */
  11. namespace Blackbit\DataDirectorBundle\EventListener;
  12. use Blackbit\DataDirectorBundle\Controller\ImportController;
  13. use Blackbit\DataDirectorBundle\lib\Pim\Import\CallbackFunction;
  14. use Blackbit\DataDirectorBundle\lib\Pim\Item\ImporterInterface;
  15. use Blackbit\DataDirectorBundle\lib\Pim\Item\ItemMoldBuilder;
  16. use Blackbit\DataDirectorBundle\lib\Pim\RawData\Importer;
  17. use Blackbit\DataDirectorBundle\lib\Pim\Serializer;
  18. use Blackbit\DataDirectorBundle\model\Dataport;
  19. use Blackbit\DataDirectorBundle\model\DataportResource;
  20. use Blackbit\DataDirectorBundle\model\Fieldmapping;
  21. use Blackbit\DataDirectorBundle\model\ImportStatus;
  22. use Blackbit\DataDirectorBundle\model\PimcoreDbRepository;
  23. use Blackbit\DataDirectorBundle\model\Queue;
  24. use Blackbit\DataDirectorBundle\model\RawItem;
  25. use Blackbit\DataDirectorBundle\model\RawItemData;
  26. use Blackbit\DataDirectorBundle\Tools\Installer;
  27. use InvalidArgumentException;
  28. use Pimcore\Db;
  29. use Pimcore\Event\Model\AssetEvent;
  30. use Pimcore\Event\Model\ElementEventInterface;
  31. use Pimcore\Logger;
  32. use Pimcore\Model\AbstractModel;
  33. use Pimcore\Model\Asset;
  34. use Pimcore\Model\DataObject\AbstractObject;
  35. use Pimcore\Model\DataObject\Concrete;
  36. use Pimcore\Model\Element\DirtyIndicatorInterface;
  37. use Pimcore\Model\Element\ElementInterface;
  38. use Pimcore\Model\Element\Service;
  39. use Pimcore\Model\User;
  40. use Pimcore\Model\Version;
  41. use Pimcore\Tool;
  42. use Blackbit\DataDirectorBundle\lib\Pim\Cli;
  43. use Psr\Log\LoggerAwareInterface;
  44. use Psr\Log\LoggerAwareTrait;
  45. use Psr\Log\LoggerInterface;
  46. use Symfony\Component\HttpFoundation\Request;
  47. class AutomaticImportListener implements LoggerAwareInterface
  48. {
  49.     use LoggerAwareTrait;
  50.     private static $processCommandRegistered = [];
  51.     /** @var ItemMoldBuilder */
  52.     private $itemMoldBuilder;
  53.     public function __construct(ItemMoldBuilder $itemMoldBuilder)
  54.     {
  55.         $this->itemMoldBuilder $itemMoldBuilder;
  56.     }
  57.     public function startImports(ElementEventInterface $e) {
  58.         if(\method_exists($e'getArgument')) {
  59.             try {
  60.                 $saveVersionOnly $e->getArgument('saveVersionOnly');
  61.                 if($saveVersionOnly) {
  62.                     return;
  63.                 }
  64.             } catch(\InvalidArgumentException $exception) {
  65.             }
  66.         }
  67.         $object $e->getElement();
  68.         $user Tool\Admin::getCurrentUser();
  69.         if($user instanceof User) {
  70.             PimcoreDbRepository::getInstance()->execute('UPDATE edit_lock SET date=? WHERE cid=? AND ctype=? AND userId=?', [time(), $object->getId(), Service::getElementType($object), $user->getId()]);
  71.         }
  72.         $dataports Dataport::getInstance();
  73.         $dataportResourceRepository DataportResource::getInstance();
  74.         $queue Queue::getInstance();
  75.         foreach($dataports->find([], 'name') as $dataport) {
  76.             $originalLocale = \Pimcore::getContainer()->get('pimcore.locale')->getLocale();
  77.             try {
  78.                 $sourceConfig unserialize($dataport['sourceconfig'], ['allowed_classes' => false]);
  79.                 $targetConfig unserialize($dataport['targetconfig'], ['allowed_classes' => false]);
  80.                 if(!empty($sourceConfig['autoImport'])) {
  81.                     $parser $dataports->getParser($dataport['id']);
  82.                     if(method_exists($parser'disableLoggingNotFoundImportResource')) {
  83.                         $parser->disableLoggingNotFoundImportResource();
  84.                     }
  85.                     if(\method_exists($parser'setSourceFile') && \method_exists($parser'getFileConditionFromObject')) {
  86.                         if (empty($targetConfig['itemClass'])) {
  87.                             // Export
  88.                             if (empty($sourceConfig['incrementalExport'])) {
  89.                                 $dataportResources $dataportResourceRepository->find(['dataportId = ?' => $dataport['id']]);
  90.                             } else {
  91.                                 $dataportResources = [
  92.                                     $dataportResourceRepository->create(
  93.                                         [
  94.                                             'dataportId' => $dataport['id'],
  95.                                             'resource' => \json_encode([], JSON_UNESCAPED_SLASHES)
  96.                                         ]
  97.                                     )
  98.                                 ];
  99.                             }
  100.                             $dataportResourceIdsStillContainingObject = [];
  101.                             foreach ($dataportResources as $dataportResource) {
  102.                                 try {
  103.                                     $resourceSettings = \json_decode($dataportResource['resource'], true);
  104.                                     $parser->setSourceFile($resourceSettings['file'] ?? null);
  105.                                     if (!empty($resourceSettings['locale'])) {
  106.                                         \Pimcore::getContainer()->get('pimcore.locale')->setLocale($resourceSettings['locale']);
  107.                                     } else {
  108.                                         $resourceSettings['locale'] = \Pimcore::getContainer()->get('pimcore.locale')->getLocale();
  109.                                         if ($resourceSettings['locale'] === null) {
  110.                                             $resourceSettings['locale'] = Tool::getDefaultLanguage();
  111.                                         }
  112.                                     }
  113.                                     $source $parser->getFileConditionFromObject($object);
  114.                                     if ($source !== null) {
  115.                                         $dataportResourceIdsStillContainingObject[$source][$resourceSettings['locale']][] = $dataportResource['id'];
  116.                                     } elseif (empty($sourceConfig['incrementalExport'])) {
  117.                                         $source $parser->getFileConditionFromObject($objectfalse);
  118.                                         if ($source !== null) {
  119.                                             $tmpSourceConfig $sourceConfig;
  120.                                             if (\method_exists($parser'setConfig')) {
  121.                                                 if (\method_exists($parser'getConfig')) {
  122.                                                     $tmpSourceConfig $parser->getConfig();
  123.                                                 }
  124.                                                 $tmpSourceConfig['file'] = '';
  125.                                                 $tmpSourceConfig['dataportId'] = $dataport['id'];
  126.                                                 $parser->setConfig($tmpSourceConfig);
  127.                                             }
  128.                                             $parser->setSourceFile($source);
  129.                                             $keyFields = [];
  130.                                             foreach ($tmpSourceConfig['fields'] as $fieldIndex => $field) {
  131.                                                 if (!empty($field['exportKey'])) {
  132.                                                     $keyFields[$fieldIndex] = $field;
  133.                                                 }
  134.                                             }
  135.                                             $hashs = [];
  136.                                             if ($dataport['sourcetype'] === 'pimcore') {
  137.                                                 $locales Tool::getValidLanguages();
  138.                                                 if ($dataportResource !== null) {
  139.                                                     $resource = \json_decode($dataportResource['resource'], true);
  140.                                                     if ($resource['locale']) {
  141.                                                         $locales = [$resource['locale']];
  142.                                                     }
  143.                                                 }
  144.                                                 foreach ($locales as $language) {
  145.                                                     \Pimcore::getContainer()->get('pimcore.locale')->setLocale($language);
  146.                                                     foreach ($parser as $rawItemData) {
  147.                                                         if ($rawItemData === null) {
  148.                                                             continue;
  149.                                                         }
  150.                                                         $rawItemData array_filter(
  151.                                                             $rawItemData,
  152.                                                             static function ($fieldId) use ($keyFields) {
  153.                                                                 return isset($keyFields[$fieldId]);
  154.                                                             },
  155.                                                             ARRAY_FILTER_USE_KEY
  156.                                                         );
  157.                                                         $hashs[] = Importer::getHash($rawItemData);
  158.                                                     }
  159.                                                 }
  160.                                             } else {
  161.                                                 foreach ($parser as $rawItemData) {
  162.                                                     if ($rawItemData === null) {
  163.                                                         continue;
  164.                                                     }
  165.                                                     $rawItemData array_filter(
  166.                                                         $rawItemData,
  167.                                                         static function ($fieldId) use ($keyFields) {
  168.                                                             return isset($keyFields[$fieldId]);
  169.                                                         },
  170.                                                         ARRAY_FILTER_USE_KEY
  171.                                                     );
  172.                                                     $hashs[] = Importer::getHash($rawItemData);
  173.                                                 }
  174.                                             }
  175.                                             if(count($hashs) > 0) {
  176.                                                 $countDeletedRawItems Db::get()->executeUpdate('DELETE FROM '.Installer::TABLE_RAWITEM.' WHERE dataport_resource_id = ? AND hash IN ("'.implode('","'$hashs).'")', [$dataportResource['id']]);
  177.                                                 if ($countDeletedRawItems) {
  178.                                                     $this->queueProcessRawData($dataportResource['id'], $dataport['id']);
  179.                                                 }
  180.                                             }
  181.                                         }
  182.                                     }
  183.                                 } catch (\Throwable $e) {
  184.                                     if(!$e instanceof SkipTriggerAutomaticImportException) {
  185.                                         $error 'Check for automatic start for dataport #'.$dataport['id'].' failed: '.(string)$e;
  186.                                         if($this->logger) {
  187.                                             $this->logger->error($error);
  188.                                         } else {
  189.                                             Logger::error($error);
  190.                                         }
  191.                                     }
  192.                                 }
  193.                             }
  194.                             foreach($dataportResourceIdsStillContainingObject as $dataportResourceQuery => $dataportResourcesWithSameLocale) {
  195.                                 foreach($dataportResourcesWithSameLocale as $locale => $dataportResourceIds) {
  196.                                     if (empty($sourceConfig['incrementalExport'])) {
  197.                                         $queue->create(
  198.                                             [
  199.                                                 'command' => 'data-director:extract '.$dataport['id'].' "'.str_replace(['"''$''`'], ['\\"''\\$''\\`'], $dataportResourceQuery).'"'.(!empty($locale) ? ' --locale='.$locale '').' --dataport-resource-id='.implode(','$dataportResourceIds),
  200.                                                 'triggered_by' => 'Element '.$object->getFullPath().' (#'.$object->getId().') got saved'.(($user instanceof User) ? ' by '.$user->getUsername() : '')
  201.                                             ]
  202.                                         );
  203.                                     } elseif ($dataport['sourcetype'] === 'pimcore' && preg_match('/ IN \(((\d+,?)+)\)/'$dataportResourceQuery$match)) {
  204.                                         $ids explode(','$match[1]);
  205.                                         sort($idsSORT_NUMERIC);
  206.                                         foreach (array_chunk($ids100) as $inIds) {
  207.                                             $queue->create(
  208.                                                 [
  209.                                                     'command' => 'data-director:complete '.$dataport['id'].' "'.str_replace([$match[0], '"''$''`'], [' IN ('.implode(',',$inIds).')''\\"''\\$''\\`'], $dataportResourceQuery).'"',
  210.                                                     'triggered_by' => 'Element '.$object->getFullPath().' (#'.$object->getId().') got saved'.(($user instanceof User) ? ' by '.$user->getUsername() : '')
  211.                                                 ]
  212.                                             );
  213.                                         }
  214.                                     } else {
  215.                                         $queue->create([
  216.                                             'command' => 'data-director:complete '.$dataport['id'].' "'.str_replace(['"''$''`'], ['\\"''\\$''\\`'], $dataportResourceQuery).'"',
  217.                                             'triggered_by' => 'Element '.$object->getFullPath().' (#'.$object->getId().') got saved'.(($user instanceof User) ? ' by '.$user->getUsername() : '')
  218.                                         ]);
  219.                                     }
  220.                                 }
  221.                             }
  222.                         } else {
  223.                             try {
  224.                                 // Import
  225.                                 $source $parser->getFileConditionFromObject($object);
  226.                                 if ($source !== null) {
  227.                                     $queue->create([
  228.                                         'command' => 'data-director:complete '.$dataport['id'].' "'.str_replace(['"''$''`'], ['\\"''\\$''\\`'], $source).'"',
  229.                                         'triggered_by' => 'Element '.$object->getFullPath().' (#'.$object->getId().') got saved'.(($user instanceof User) ? ' by '.$user->getUsername() : '')
  230.                                     ]);
  231.                                 }
  232.                             } catch(SkipTriggerAutomaticImportException $e) {
  233.                             }
  234.                         }
  235.                     }
  236.                 }
  237.             } catch (\Exception $e) {
  238.                 if($this->logger) {
  239.                     $this->logger->error((string)$e);
  240.                 } else {
  241.                     Logger::error((string)$e);
  242.                 }
  243.             } finally {
  244.                 \Pimcore::getContainer()->get('pimcore.locale')->setLocale($originalLocale);
  245.             }
  246.         }
  247.     }
  248.     public function deleteRawdata(ElementEventInterface $e) {
  249.         if(\method_exists($e'getArgument')) {
  250.             try {
  251.                 $saveVersionOnly $e->getArgument('saveVersionOnly');
  252.                 if($saveVersionOnly) {
  253.                     return;
  254.                 }
  255.             } catch(\InvalidArgumentException $exception) {
  256.             }
  257.         }
  258.         $object $e->getElement();
  259.         $dataports Dataport::getInstance();
  260.         $objectType Service::getElementType($object);
  261.         $commandPrefix ' "'.Cli::getPhpCli().'" '.realpath(PIMCORE_PROJECT_ROOT.DIRECTORY_SEPARATOR.'bin'.DIRECTORY_SEPARATOR.'console');
  262.         foreach($dataports->find([], 'name') as $dataport) {
  263.             $sourceConfig unserialize($dataport['sourceconfig'], ['allowed_classes' => false]);
  264.             $targetConfig unserialize($dataport['targetconfig'], ['allowed_classes' => false]);
  265.             if(!empty($sourceConfig['autoImport']) && empty($targetConfig['itemClass'])) {
  266.                 $itemMold $this->itemMoldBuilder->getItemMoldByClassId($sourceConfig['sourceClass']);
  267.                 if($object instanceof $itemMold) {
  268.                     chdir(PIMCORE_PROJECT_ROOT);
  269.                     Cli::exec($commandPrefix.' data-director:delete-rawdata --dataport='.$dataport['id'].' --object-id='.$object->getId().' --object-type='.$objectType);
  270.                 }
  271.             }
  272.         }
  273.     }
  274.     private function queueDeleteRawdata(ElementInterface $object$dataportResourceId null) {
  275.         $queue Queue::getInstance();
  276.         $objectType Service::getElementType($object);
  277.         $queue->create(['command' => 'data-director:delete-rawdata --dataport-resource-id=' .$dataportResourceId ' --object-id=' $object->getId() . ' --object-type=' $objectType'triggered_by' => 'Item '.$object->getFullPath().' does not fulfill the dataport\'s SQL condition anymore']);
  278.     }
  279.     private function queueProcessRawData($dataportResourceId$dataportId) {
  280.         if (!isset(self::$processCommandRegistered[$dataportResourceId])) {
  281.             self::$processCommandRegistered[$dataportResourceId] = true;
  282.             $queue Queue::getInstance();
  283.             $cmd 'data-director:process ' $dataportId ' --dataport-resource-id=' $dataportResourceId;
  284.             register_shutdown_function(
  285.                 static function () use ($queue$cmd) {
  286.                     $queue->create(['command' => $cmd'triggered_by' => 'An item got deleted from raw data -> new result document has to be generated']);
  287.                 }
  288.             );
  289.         }
  290.     }
  291. }