Adds new command to send fake AI Bot Requests

Commands/GenerateBotRequests.php

@@ -0,0 +1,132 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\VisitorGenerator\Commands;
+
+use Piwik\Access;
+use Piwik\Date;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Plugins\VisitorGenerator\Generator\BotRequestsFake;
+use Piwik\Site;
+use Piwik\Timer;
+use Piwik\UrlHelper;
+
+class GenerateBotRequests extends ConsoleCommand
+{
+    protected function configure()
+    {
+        $this->setName('visitorgenerator:generate-bot-requests');
+        $this->setDescription('Generates many bot requests for a given amount of days in the past. This command is intended for developers.');
+        $this->addRequiredValueOption('idsite', null, 'Defines the site the requests should be generated for');
+        $this->addRequiredValueOption('days', null, 'Defines for how many days in the past requests should be generated', 1);
+        $this->addRequiredValueOption('start-date', null, 'Date to start generating on.');
+        $this->addRequiredValueOption('limit-fake-requests', null, 'Limits the number of fake requests', null);
+        $this->addRequiredValueOption('custom-matomo-url', null, "Defines an alternate Matomo URL, e.g. if Matomo is installed behind a Load-Balancer.");
+    }
+
+    /**
+     * @return int
+     */
+    protected function doExecute(): int
+    {
+        $input  = $this->getInput();
+        $output = $this->getOutput();
+        $timer  = new Timer();
+        $days   = $this->checkDays();
+        $customMatomoUrl = $this->checkCustomMatomoUrl();
+        $limit  = $this->getLimitFakeRequests();
+        $idSite = $this->getIdSite();
+
+        $startDate = $input->getOption('start-date');
+        if (empty($startDate) || $startDate === 'now') {
+            $startTime = time();
+        } else {
+            $startTime = strtotime($startDate);
+        }
+
+        $time = $startTime - ($days - 1) * 86400;
+
+        $nbActionsTotal = 0;
+        while ($time <= $startTime) {
+            $output->writeln([
+                sprintf("Generating requests for %s...", Date::factory($time)->toString()),
+            ]);
+
+            Access::doAsSuperUser(function () use ($time, $idSite, &$nbActionsTotal, $customMatomoUrl, $limit) {
+                $fakeRequests = new BotRequestsFake($customMatomoUrl);
+                $nbActionsTotal += $fakeRequests->generate($time, $idSite, $limit);
+            });
+
+            $time += 86400;
+        }
+
+        $this->writeSuccessMessage([
+            'idsite = ' . $idSite,
+            $nbActionsTotal . ' Requests generated',
+            round($nbActionsTotal / $timer->getTime(), 0) . ' requests per second',
+        ]);
+
+        return self::SUCCESS;
+    }
+
+    private function getLimitFakeRequests(): int
+    {
+        $input = $this->getInput();
+
+        if ($input->getOption('limit-fake-requests')) {
+            return $input->getOption('limit-fake-requests');
+        }
+
+        return rand(400, 1000);
+    }
+
+    protected function checkDays(): int
+    {
+        $days = (int)$this->getInput()->getOption('days');
+
+        if ($days < 1) {
+            throw new \InvalidArgumentException('Days to compute must be greater or equal to 1.');
+        }
+
+        return $days;
+    }
+
+    private function checkCustomMatomoUrl(): ?string
+    {
+        if (!$customMatomoUrl = $this->getInput()->getOption('custom-matomo-url')) {
+            return null;
+        }
+
+        if (!UrlHelper::isLookLikeUrl($customMatomoUrl)) {
+            throw new \Exception("The Custom Matomo Tracker Url you entered doesn't seem to be valid.");
+        }
+
+        return $customMatomoUrl;
+    }
+
+    protected function getIdSite(): int
+    {
+        $idSite = $this->getInput()->getOption('idsite');
+
+        if ($idSite === null) {
+            $idSite = $this->ask('ID of the site in which to generate the visits: ');
+        }
+
+        $idSite = (int)$idSite;
+
+        return Access::doAsSuperUser(function () use ($idSite) {
+            if (!Site::getSite($idSite)) {
+                throw new \InvalidArgumentException('idsite is not a valid, no such site found');
+            }
+
+            return $idSite;
+        });
+    }
+}

Faker/Request.php

@@ -767,7 +767,7 @@ class Request extends \Faker\Provider\Base
         '/blog/tasks/',
         '/blog/2013/09/piwik-mobile-1-9-8-with-full-support-for-ios-7/',
         '/FAQ/',
-        '/blog/tasks/graphs-list-of-improvements-jqplotjs/'
+        '/blog/tasks/graphs-list-of-improvements-jqplotjs/',
     );
 
     protected static $campaignNames = [
@@ -849,7 +849,7 @@ class Request extends \Faker\Provider\Base
         '2048x1152',
         '600x912',
         '320x240',
-        '320x568'
+        '320x568',
     );
 
     public function pageURL()
@@ -878,7 +878,7 @@ public function productName()
             'Test Product',
             'Product Name',
             'Awesome Product',
-            'Helpful Product'
+            'Helpful Product',
         ));
     }
 
@@ -981,7 +981,7 @@ public function searchEngineUrlWithKeyword($keyword)
             'http://www.bing.com/search?q=',
             'http://uk.search.yahoo.com/search?ei=UTF-8&fr=moz35&q=',
             'http://search.yahoo.com/search;?p=',
-            'http://uk.ask.com/web?l=dis&prt=NIS&chn=retail&geo=GB&ver=19&odn=&dm=all&q='
+            'http://uk.ask.com/web?l=dis&prt=NIS&chn=retail&geo=GB&ver=19&odn=&dm=all&q=',
         ));
 
         return $url . $keyword;
@@ -998,7 +998,7 @@ public function searchEngineUrlWithOutKeyword()
             'http://search.yahoo.co.jp/r/FOR=wJjKBrpV3iiyQ9OVtYIAEFhwB6F4X7nB3uwubUjurHLqwT8oFNO7dNiZBXYbmgNENZz8yKntTR'
                 . '5X_chrfaAcSOgPUyuK.Y5mOUs5tzKNLuq42vgCKLHdBH3gavAtCoKeCKOItwbkCN10nMDO9gO8MW9zdH9TOyAul7u1sqXEE0wTNp'
                 . 'Y6DW.EFfagOL54JzI3OBVfIin7/_ylt=A2RAyhx.wOZaQWsATEuDTwx.;_ylu=X3oDMTBtNHJhZXRnBHBvcwMxBHNlYwNzcgRzbGsDdGl0bGU-/SIG=11fhbagaq/EXP=1525172798/**https%3A//matomo.jp/news/7540',
-            'https://yandex.ru/'
+            'https://yandex.ru/',
         ));
 
         return $url;
@@ -1102,7 +1102,20 @@ public function referrer()
             'http://www.example.com/5-great-google-analytics-alternatives/?utm_content=ourfake&utm_medium=social&utm_source=twitter.com&utm_campaign=buffer',
             'http://www.example.org/blog/15-google-analytics-alternatives?utm_source=twitter&utm_medium=social&utm_campaign=blog',
             'http://t.co/itFSLvPlqW',
-            'http://t.co/g0Alwc83T6'
+            'http://t.co/g0Alwc83T6',
             ));
     }
+
+    public function aiAssistantBotUserAgent()
+    {
+        return static::randomElement([
+            'Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com)',
+            'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)',
+            'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36',
+            'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)',
+            'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)',
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36; Devin/1.0; +devin.ai',
+            'Google-NotebookLM',
+        ]);
+    }
 }

Generator/BotRequestsFake.php

@@ -0,0 +1,104 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Plugins\VisitorGenerator\Generator;
+
+use Piwik\Piwik;
+use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeApi;
+use Piwik\Plugins\SitesManager\API as SitesManagerApi;
+use Piwik\Plugins\VisitorGenerator\Generator;
+
+class BotRequestsFake extends Generator
+{
+    public function generate(int $time = null, int $idSite = 1, int $limit = 1000): int
+    {
+        $date = date("Y-m-d", $time);
+
+        $tracker = $this->makeMatomoTracker($idSite);
+        $tracker->enableBulkTracking();
+        $tokenAuth = Piwik::requestTemporarySystemAuthToken('VistorGenerator', 24);
+        $site = $this->getCurrentSite($idSite);
+
+        $i = 0;
+        while ($i < $limit) {
+            $tracker->setNewVisitorId();
+
+            $pageUrl = $this->faker->pageURL;
+
+            // don't use link urls for bot requests
+            while (0 === strpos($pageUrl, 'http')) {
+                $pageUrl = $this->faker->pageURL;
+            }
+
+            $i++;
+            $tracker->setTokenAuth($tokenAuth);
+            $tracker->setUserAgent($this->faker->aiAssistantBotUserAgent);
+
+            $tracker->setIp($this->faker->boolean(77) ? $this->faker->ipv4 : $this->faker->ipv6);
+            $tracker->setLocalTime($this->faker->time());
+            $tracker->setIdSite($idSite);
+
+            $tracker->setPerformanceTimings(
+                null,
+                $this->faker->numberBetween(400, 600),
+                null,
+                null,
+                null,
+                null
+            );
+
+            $tracker->setForceVisitDateTime($date . ' ' . $this->faker->time('H:i:s'));
+
+            $tracker->setCustomTrackingParameter('bw_bytes', $this->faker->numberBetween(1, 2000000000));
+            $tracker->setCustomTrackingParameter('http_status', $this->faker->randomElement([
+                // most requests should be status 200, so adding it here more often
+                200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200,
+                200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200,
+                301, 302, 307, 308,
+                401, 404, 404, 404, 404,
+                500,
+            ]));
+
+            $tracker->setCustomTrackingParameter('source', $this->faker->randomElement([
+                '', 'cloudflare', 'cloudfront', 'wordpress'
+            ]));
+
+            // force bot tracking mode
+            $tracker->setCustomTrackingParameter('recMode', '1');
+
+
+            if (false !== strpos($pageUrl, 'zip')) {
+                $tracker->setUrl($site['main_url'] . $pageUrl);
+                $tracker->doTrackAction($site['main_url'] . $pageUrl, 'download');
+            } else {
+                $tracker->setUrl($site['main_url'] . $pageUrl);
+
+                $tracker->doTrackPageView('');
+            }
+
+            if ($i % 100 == 0) {
+                $tracker->doBulkTrack();
+            }
+        }
+
+        if ($i % 100 != 0) {
+            $tracker->doBulkTrack();
+        }
+
+        CoreAdminHomeApi::getInstance()->invalidateArchivedReports($idSite, $date);
+
+        return $i;
+    }
+
+    private function getCurrentSite($idSite)
+    {
+        return SitesManagerApi::getInstance()->getSiteFromId($idSite);
+    }
+}