Merge pull request #1604 from griidc/release/6.59.1

release/6.59.1

.env

@@ -46,17 +46,9 @@ ACCOUNT_LESS_STRICT_PASSWORD_RULES=0
 MAILER_FROM_ADDR='[email protected]'
 MAILER_FROM_NAME='SenderName'
 MAILER_BCC_ADDR='[email protected]'
-PELAGOS_NAME=pelagos
 ISSUE_TRACKING_BASE_URL='https://griidc.atlassian.net/browse'
 DATABASE_NAME=pelagos
 DATA_STORE_DIRECTORY=/path/to/store/directory
-DATA_DOWNLOAD_DIRECTORY=/path/to/download/directory
-DATA_STORE_OWNER=user
-DATA_STORE_GROUP=group
-DATA_DOWNLOAD_BROWSER_GROUP=browser_group
-WEB_SERVER_USER=apache
-ANONYMOUS_FTP_USER=user
-ANONYMOUS_FTP_PASS=pass
 DOWNLOAD_PATH="path/to/zip"
 INGEST_API_URL="https://ingest-server:port/api-name"
 GCMD_VERSION="16.9"
@@ -77,7 +69,6 @@ DOWNLOAD_BASE_URL=https://server-name/path/file-download
 POSIX_STARTING_UID=10000
 POSIX_GID=1001
 REQUEST_CONTEXT_BASE_URL='/pelagos-symfony'
-FILESYSTEM_TYPE='Linux'
 
 ###> DOI API Parameters ###
 DOI_API_USER_NAME="username"

config/services.yaml

@@ -94,6 +94,7 @@ services:
             - { name: kernel.event_listener, event: pelagos.logactionitem.search_terms_log, method: onNewLogCreated }
             - { name: kernel.event_listener, event: pelagos.logactionitem.restrictions_log, method: onNewLogCreated }
             - { name: kernel.event_listener, event: pelagos.logactionitem.remotelyhosted_update_log, method: onNewLogCreated }
+            - { name: kernel.event_listener, event: pelagos.logactionitem.dataset_deletion, method: onNewLogCreated }
 
     App\Event\InformationProductListener:
         tags:

src/Command/GetGoMRIStatisticsCommand.php

@@ -27,6 +27,8 @@ class GetGoMRIStatisticsCommand extends Command
     public const HARVEST2019COUNT = 7100; // count
     public const HARVEST2019DATA = 3600; // GB
 
+    public const NUMBEROFTOPDOWNLOADSTOSHOW = 10;
+
     /**
      * Class constructor for dependency injection.
      */
@@ -102,6 +104,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         sort($years);
         $minYear = $years[0];
         $maxYear = $years[sizeof($years) - 1];
+        $io->writeln("\nGomri Uploads (submissions)\n");
         for ($i = $minYear; $i <= $maxYear; $i++) {
             if (in_array($i, $years)) {
                 $io->writeln("Number of GoMRI datasets submitted $i"
@@ -112,18 +115,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
-        $io->writeln("Total GoMRI Downloads:");
-        $downloadSizeByYear = [];
-        $downloadCountByYear = [];
+        $io->writeln("\nGoMRI Downloads:\n");
+        $downloadSizeByYearAndQuarter = [];
+        $downloadCountByYearAndQuarter = [];
         foreach ($this->getDownloads() as $datasetDownload) {
             $id = $datasetDownload[0];
             $timestamp = $datasetDownload[1];
-            $year = substr($timestamp, 0, 4);
-            if (!array_key_exists($year, $downloadCountByYear)) {
-                $downloadCountByYear[$year] = 0;
+
+            $yearQuarter = $this->determineQuarter($timestamp);
+            $year = $yearQuarter['year'];
+            $quarter = $yearQuarter['quarter'];
+
+            if (!array_key_exists($year, $downloadCountByYearAndQuarter)) {
+                $downloadCountByYearAndQuarter[$year][0] = 0;
+                $downloadCountByYearAndQuarter[$year][1] = 0;
+                $downloadCountByYearAndQuarter[$year][2] = 0;
+                $downloadCountByYearAndQuarter[$year][3] = 0;
+                $downloadCountByYearAndQuarter[$year][4] = 0;
             }
-            if (!array_key_exists($year, $downloadSizeByYear)) {
-                $downloadSizeByYear[$year] = 0;
+
+            if (!array_key_exists($year, $downloadSizeByYearAndQuarter)) {
+                $downloadSizeByYearAndQuarter[$year][0] = 0;
+                $downloadSizeByYearAndQuarter[$year][1] = 0;
+                $downloadSizeByYearAndQuarter[$year][2] = 0;
+                $downloadSizeByYearAndQuarter[$year][3] = 0;
+                $downloadSizeByYearAndQuarter[$year][4] = 0;
             }
 
             $dataset = $this->entityManager->find('\App\Entity\Dataset', $id);
@@ -136,54 +152,102 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 $skipCount++;
             }
 
-
-            $downloadCountByYear[$year]++;
-            $downloadSizeByYear[$year] += $size / 1000000000;
-            //print "$timestamp,$id,$udi,$size\n";
+            $downloadCountByYearAndQuarter[$year][$quarter]++;
+            $downloadSizeByYearAndQuarter[$year][$quarter] += $size / 1000000000;
         }
 
-        $firstYear = min(array_keys($downloadCountByYear));
-        $lastYear = max(array_keys($downloadCountByYear));
-
-        // Deal with the data harvest of 2019.
-        if (array_key_exists(2019, $downloadCountByYear)) {
-            $downloadCountByYear[2019] -= self::HARVEST2019COUNT;
-        }
-        if (array_key_exists(2019, $downloadCountByYear)) {
-            $downloadSizeByYear[2019] -= self::HARVEST2019DATA;
-        }
+        // array_keys still onlys dump first level, years, for $array[year][quarter].
+        $firstYear = min(array_keys($downloadCountByYearAndQuarter));
+        $lastYear = max(array_keys($downloadCountByYearAndQuarter));
+
+        // Deal with the data harvest of 2019Q2.
+        $downloadCountByYearAndQuarter[2019][2] -= self::HARVEST2019COUNT;
+        $downloadSizeByYearAndQuarter[2019][2] -= self::HARVEST2019DATA;
+
+        $totalDownloads = 0;
+        $totalDownloadSize = 0;
+        for ($year = $firstYear; $year <= $lastYear; $year++) {
+            $yearCountTotal = 0;
+            $yearSizeTotal = 0;
+            for ($quarter = 1; $quarter <= 4; $quarter++) {
+                $popularDownloads = $this->getTopDatasetsDownloadedByYearAndQuarter(self::NUMBEROFTOPDOWNLOADSTOSHOW, $year, $quarter);
+                $io->writeln($year . '/Q' . $quarter . ' Download Count: ' . $downloadCountByYearAndQuarter[$year][$quarter]
+                . ', ' . 'Total Size (GB): ' . round($downloadSizeByYearAndQuarter[$year][$quarter]));
+                $yearCountTotal += $downloadCountByYearAndQuarter[$year][$quarter];
+                $totalDownloads += $downloadCountByYearAndQuarter[$year][$quarter];
+                $yearSizeTotal += $downloadSizeByYearAndQuarter[$year][$quarter];
+                $totalDownloadSize += $downloadSizeByYearAndQuarter[$year][$quarter];
+                $popular = "Top: ";
+                foreach ($popularDownloads as $udi => $count) {
+                    $popular .= "$udi:$count, ";
+                }
+                $popular = substr($popular, 0, strlen($popular) - 2);
+                $io->writeln($popular);
+            }
+            $io->newLine();
 
+            $io->writeln('Totals for: ' . $year . ':');
+            $io->writeln('Download count: ' . $yearCountTotal);
+            $io->writeln('Data Downloaded: ' . round($yearSizeTotal) . ' GB');
+            $io->writeln('--------------------------------------------------------------------------------');
 
+            $io->newLine();
+        }
 
-        for ($i = $firstYear; $i <= $lastYear; $i++) {
-            $io->writeln($i . ' Download Count: ' . $downloadCountByYear[$i] . ', ' . 'Total Size (GB): ' . round($downloadSizeByYear[$i]));
+        // Total Download stats:
+        $io->writeln('All Time Stats:');
+        $io->writeln('Download count: ' . $totalDownloads);
+        $io->writeln('Data Downloaded: ' . round($totalDownloadSize) . ' GB');
+
+        // Show most popular downloads of all time.
+        $popularDownloadsOfAllTime = $this->getTopDatasetsDownloadedByYearAndQuarter(self::NUMBEROFTOPDOWNLOADSTOSHOW);
+        $allTimePopular = '';
+        $io->writeln('Top ' . self::NUMBEROFTOPDOWNLOADSTOSHOW . ' downloads of all time:');
+        foreach ($popularDownloadsOfAllTime as $udi => $count) {
+            $allTimePopular .= "$udi:$count, ";
         }
+        $allTimePopular = substr($allTimePopular, 0, strlen($allTimePopular) - 2);
+        $io->writeln($allTimePopular);
+
+        // Add disclaimer if any datasets had been deleted. (This doesn't apply to gomri currently, but other FOs)
         if ($skipCount > 0) {
             $io->warning("Skipped $skipCount entries as these datasets are no longer available.");
         }
 
-        $io->note('Removed 7100 / 3600GB from 2019 downloads - data harvest occurred');
+        // Add disclaimer for data cleanup for big data harvest event.
+        $io->note('Removed ' . self::HARVEST2019COUNT . ' / ' . self::HARVEST2019DATA . 'GB from 2019Q2 downloads - data harvest occurred');
 
         return 0;
     }
 
     /**
-     * Maintains array of quarter counts.
+     * Returns Year and Quarter given a timestamp.
      *
      * @param \DateTime $timestamp
-     * @param array $quarterCounts
+     * @return array $yearQuarter
      */
-    protected function quarterize(\DateTime $timestamp, array &$quarterCounts): void
+    protected function determineQuarter(\DateTime $timestamp): array
     {
-        // Quarters are strict calendar quarters.
-        // Q1: Jan 1 - Mar 31
-        // Q2: Apr 1 - Jun 30
-        // Q3: Jul 1 - Sep 30
-        // Q4: Oct 1 - Dec 31
+        // Quarters are normal calendar quarters (Jan-Mar, Apr-Jun, Jul-Sep, Oct-Dec)
         $year = $timestamp->format('Y');   // returns 4 digit year
         $month = $timestamp->format('n');  // returns 1-12
         $quarter = ceil($month / 3);
 
+        return array("year" => $year, "quarter" => $quarter);
+    }
+
+    /**
+     * Maintains array of quarter counts.
+     *
+     * @param \DateTime $timestamp
+     * @param array $quarterCounts
+     */
+    protected function quarterize(\DateTime $timestamp, array &$quarterCounts): void
+    {
+        $yearAndQuarter = $this->determineQuarter($timestamp);
+        $year = $yearAndQuarter['year'];
+        $quarter = $yearAndQuarter['quarter'];
+
         // Initialize
         if (!array_key_exists($year, $quarterCounts)) {
             $quarterCounts[$year] = [];
@@ -196,6 +260,88 @@ protected function quarterize(\DateTime $timestamp, array &$quarterCounts): void
         $quarterCounts[$year][$quarter - 1] += 1;
     }
 
+    /**
+     * Returns array of UDIs of top downloads in date range.
+     *
+     * @param  int|null   $year            The year to get top downloads from.
+     * @param  int|null   $quarter         The quarter to get top downloads from.
+     * @return array      $topDownloadUdis An associative array of top UDIs with counts as value.
+     *
+     * @throws \Exception If quarter value used other than (1, 2, 3, 4)
+     */
+    public function getTopDatasetsDownloadedByYearAndQuarter(int $count, int $year = null, int $quarter = null): array
+    {
+        if ($quarter !== null && !(in_array($quarter, array(1, 2, 3, 4)))) {
+            throw new \Exception("Bad quarter specified, use 1-4.");
+        }
+
+        if ($year === null && $quarter !== null) {
+            throw new \Exception("If quarter is specified, year must be too.");
+        }
+
+        // Create DB compatible strings from DateTime objects.
+        if ($year !== null && $quarter == 1) {
+            $from = "$year-01-01";
+            $to = "$year-03-31";
+        } elseif ($year !== null && $quarter == 2) {
+            $from = "$year-04-01";
+            $to = "$year-06-30";
+        } elseif ($year !== null && $quarter == 3) {
+            $from = "$year-07-01";
+            $to = "$year-09-30";
+        } elseif ($year !== null && $quarter == 4) {
+            $from = "$year-10-01";
+            $to = "$year-12-31";
+        } elseif ($year !== null && $quarter === null) {
+            // entire year
+            $from = "$year-01-01";
+            $to = "$year-12-31";
+        }
+
+        $qb = $this->entityManager->getRepository(LogActionItem::class)->createQueryBuilder('log')
+        ->select('count(log.subjectEntityId), log.subjectEntityId')
+        ->where('log.subjectEntityName = :entityName')
+        ->andWhere('log.actionName = :actionName')
+        ->orderBy('count(log.subjectEntityId)', 'DESC')
+        ->groupBy('log.subjectEntityId')
+        ->setMaxResults($count)
+        ->setParameter('entityName', 'Pelagos\Entity\Dataset')
+        ->setParameter('actionName', 'File Download');
+
+        if ($year !== null) {
+            $qb
+            ->andWhere('log.creationTimeStamp >= :from')
+            ->andWhere('log.creationTimeStamp <= :to')
+            ->setParameter('from', $from)
+            ->setParameter('to', $to);
+        }
+
+        if ($this->fundingOrgFilter->isActive()) {
+            $researchGroupIds = $this->fundingOrgFilter->getResearchGroupsIdArray();
+
+            $qb
+            ->join(Dataset::class, 'dataset', Query\Expr\Join::WITH, 'log.subjectEntityId = dataset.id')
+            ->innerJoin('dataset.researchGroup', 'rg')
+            ->andWhere('rg.id IN (:rgs)')
+            ->setParameter('rgs', $researchGroupIds);
+        }
+
+        $query = $qb->getQuery();
+        $results = $query->getResult();
+
+        $topDownloadUdis = [];
+        foreach ($results as $row) {
+            $count = $row[1];
+            $id = $row['subjectEntityId'];
+            $dataset = $this->entityManager->find('\App\Entity\Dataset', $id);
+            if ($dataset instanceof Dataset) {
+                $udi = $dataset->getUdi();
+                $topDownloadUdis[$udi] = $count;
+            }
+        }
+        return $topDownloadUdis;
+    }
+
     /**
      * Generates array of funding-org filter-aware download events.
      *
@@ -236,7 +382,7 @@ public function getDownloads(): array
 
             if (($displayTime === '2014-09-27') or $key === array_key_first($downloads) or ($epochTime - $currentTimeStamp) > 30 or $currentId != $id) {
                 $currentTimeStamp = $epochTime;
-                $downloadArray[] = array($id, $displayTime);
+                $downloadArray[] = array($id, $dateTime);
             }
             $currentId = $id;
         }

src/Controller/Admin/DashboardController.php

@@ -5,6 +5,7 @@
 use App\Entity\DigitalResourceTypeDescriptor;
 use App\Entity\Funder;
 use App\Entity\FundingOrganization;
+use App\Entity\LogActionItem;
 use App\Entity\ProductTypeDescriptor;
 use EasyCorp\Bundle\EasyAdminBundle\Config\Dashboard;
 use EasyCorp\Bundle\EasyAdminBundle\Config\MenuItem;
@@ -45,6 +46,8 @@ public function configureDashboard(): Dashboard
     public function configureMenuItems(): iterable
     {
         yield MenuItem::linktoDashboard('Admin Links', 'fa fa-link');
+        yield MenuItem::section('Viewers');
+        yield MenuItem::linkToCrud('Logged Actions', 'fas fa-search', LogActionItem::class);
         yield MenuItem::section('Editors');
         yield MenuItem::linkToCrud('IP Product Descriptor', 'fas fa-list-alt', ProductTypeDescriptor::class);
         yield MenuItem::linkToCrud('IP Digital Resource Descriptor', 'fas fa-list-alt', DigitalResourceTypeDescriptor::class);

src/Controller/Admin/LogActionItemCrudController.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Controller\Admin;
+
+use App\Entity\LogActionItem;
+use EasyCorp\Bundle\EasyAdminBundle\Config\Action;
+use EasyCorp\Bundle\EasyAdminBundle\Config\Actions;
+use EasyCorp\Bundle\EasyAdminBundle\Config\Crud;
+use EasyCorp\Bundle\EasyAdminBundle\Controller\AbstractCrudController;
+use EasyCorp\Bundle\EasyAdminBundle\Field\DateTimeField;
+use EasyCorp\Bundle\EasyAdminBundle\Field\IdField;
+use EasyCorp\Bundle\EasyAdminBundle\Field\TextField;
+
+class LogActionItemCrudController extends AbstractCrudController
+{
+    public static function getEntityFqcn(): string
+    {
+        return LogActionItem::class;
+    }
+
+    /**
+     * Configure the Crud actions.
+     *
+     * @param Actions $actions actions object that need to be configured
+     */
+    public function configureActions(Actions $actions): Actions
+    {
+        $actions->disable(Action::NEW);
+        $actions->disable(Action::EDIT);
+        $actions->disable(Action::DELETE);
+
+        return $actions;
+    }
+
+    /**
+     * Configure fields for EZAdmin CRUD Controller.
+     *
+     * @param string $pageName default param for parent method (not used)
+     */
+    public function configureFields(string $pageName): iterable
+    {
+        return [
+            IdField::new('id')
+            ->onlyOnIndex(),
+            TextField::new('actionName', 'What Happened'),
+            TextField::new('subjectEntityName', 'Happened To'),
+            IdField::new('subjectEntityId', 'Which One'),
+            DateTimeField::new('creationTimeStamp', 'Happened When'),
+            TextField::new('payloadDetails'),
+        ];
+    }
+
+    /**
+     * CRUD configuration function.
+     *
+     * @param Crud $crud instance for crud controller to add additional configuration
+     */
+    public function configureCrud(Crud $crud): Crud
+    {
+        return parent::configureCrud($crud)
+            ->setPageTitle(Crud::PAGE_INDEX, 'Logged Actions')
+            ->setDefaultSort(['creationTimeStamp' => 'DESC']);
+    }
+}