Skip to content

Commit 2894f16

Browse files
sacherasachera
authored
Merge pull request #47 from Flowpack/log-statistics-events
FEATURE: Statistics events logging
2 parents e525968 + 1f6b72f commit 2894f16

10 files changed

Lines changed: 268 additions & 11 deletions

‎Classes/BackendUi/BackendUiDataService.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Repository\RedisEnumerationRepository;
1414
use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
1515
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager;
16-
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore;
16+
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingTimeStatisticsStore;
1717
use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
1818
use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService;
1919
use Neos\Flow\Annotations as Flow;
@@ -44,7 +44,7 @@ class BackendUiDataService
4444

4545
/**
4646
* @Flow\Inject
47-
* @var RedisRenderingStatisticsStore
47+
* @var RedisRenderingTimeStatisticsStore
4848
*/
4949
protected $redisRenderingStatisticsStore;
5050

‎Classes/Command/ContentReleaseEventsCommandController.php‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace Flowpack\DecoupledContentStore\Command;
5+
6+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId;
7+
use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisStatisticsEventService;
8+
use Neos\Flow\Annotations as Flow;
9+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
10+
use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
11+
use Neos\Flow\Cli\CommandController;
12+
13+
/**
14+
* Commands to read statistics events for a content release from redis
15+
*/
16+
class ContentReleaseEventsCommandController extends CommandController
17+
{
18+
#[Flow\Inject]
19+
protected RedisStatisticsEventService $redisStatisticsEventService;
20+
21+
/**
22+
* Count statistics events in a content release.
23+
*
24+
* This command will count how many statics events with given filters, grouped by the specified keys exist in a content release.
25+
*
26+
* Use --where to filter the events (e.g. "--where=event=title") and --groupBy to count events separately
27+
* (e.g. "--groupBy=additionalPayload.preset"). You can specify multiple filters and groups by separating them with
28+
* ',' (e.g "--groupBy=additionalPayloads.preset,additionalPayloads.documentId")
29+
*
30+
* Do not use "--where event=title"! Flow will remove "event=" and the filter will not be applied.
31+
*
32+
* @param string $contentReleaseIdentifier The contentReleaseIdentifier which statistics should be counted
33+
* @param string $where filter the events before counting
34+
* @param string $groupBy group the events by this value into separately counted groups
35+
* @return void
36+
*/
37+
public function countStatisticsEventCommand(string $contentReleaseIdentifier, string $where = '', string $groupBy = ''): void
38+
{
39+
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
40+
// split every string in $where by the first '=' and use the left part as key and the right part as value
41+
$where = $where ? array_column(array_map(fn($s) => explode('=', $s, 2), explode(',', $where)), 1, 0) : [];
42+
$groupBy = $groupBy ? explode(',', $groupBy) : [];
43+
44+
$this->output("Filters: \n");
45+
if($where) {
46+
foreach ($where as $key=>$value) {
47+
$this->output(" $key = \"$value\"\n");
48+
}
49+
} else {
50+
$this->output(" None \n");
51+
}
52+
53+
$eventCounts = $this->redisStatisticsEventService->countEvents($contentReleaseIdentifier, $where, $groupBy);
54+
$this->output->outputTable($eventCounts, array_merge(['count'], $groupBy));
55+
56+
$this->output("Total: %d\n", [array_sum(array_map(fn($e) => $e['count'], $eventCounts))]);
57+
}
58+
}

‎Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
5+
6+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
7+
use Neos\Flow\Cli\ConsoleOutput;
8+
use Symfony\Component\Console\Output\OutputInterface;
9+
10+
class ConsoleStatisticsEventOutput implements StatisticsEventOutputInterface
11+
{
12+
protected OutputInterface $output;
13+
14+
function __construct(OutputInterface $output)
15+
{
16+
$this->output = $output;
17+
}
18+
19+
public static function fromConsoleOutput(ConsoleOutput $output): self
20+
{
21+
return static::fromSymfonyOutput($output->getOutput());
22+
}
23+
24+
public static function fromSymfonyOutput(OutputInterface $output): self
25+
{
26+
return new static($output);
27+
}
28+
29+
public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
30+
{
31+
$this->output->writeln($prefix . 'STATISTICS EVENT ' . $event . ($additionalPayload ? ' ' . json_encode($additionalPayload) : ''));
32+
}
33+
}

‎Classes/Core/Infrastructure/ContentReleaseLogger.php‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ class ContentReleaseLogger
1414
*/
1515
protected $output;
1616

17+
/**
18+
* @var StatisticsEventOutputInterface
19+
*/
20+
protected $statisticsEventOutput;
21+
1722
/**
1823
* @var ContentReleaseIdentifier
1924
*/
@@ -24,10 +29,11 @@ class ContentReleaseLogger
2429
*/
2530
protected $logPrefix = '';
2631

27-
protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, ?RendererIdentifier $rendererIdentifier)
32+
protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput, ?RendererIdentifier $rendererIdentifier)
2833
{
2934
$this->output = $output;
3035
$this->contentReleaseIdentifier = $contentReleaseIdentifier;
36+
$this->statisticsEventOutput = $statisticsEventOutput;
3137
$this->rendererIdentifier = $rendererIdentifier;
3238
$this->logPrefix = '';
3339

@@ -37,14 +43,14 @@ protected function __construct(OutputInterface $output, ContentReleaseIdentifier
3743
}
3844

3945

40-
public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier): self
46+
public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self
4147
{
42-
return new static($output->getOutput(), $contentReleaseIdentifier, null);
48+
return new static($output->getOutput(), $contentReleaseIdentifier, $statisticsEventOutput, null);
4349
}
4450

45-
public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier): self
51+
public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self
4652
{
47-
return new static($output, $contentReleaseIdentifier, null);
53+
return new static($output, $contentReleaseIdentifier, $statisticsEventOutput, null);
4854
}
4955

5056
public function debug($message, array $additionalPayload = [])
@@ -72,8 +78,13 @@ public function logException(\Exception $exception, string $message, array $addi
7278
$this->output->writeln($this->logPrefix . $message . "\n\n" . $exception->getMessage() . "\n\n" . $exception->getTraceAsString() . "\n\n" . json_encode($additionalPayload));
7379
}
7480

81+
public function logStatisticsEvent(string $event, array $additionalPayload = [])
82+
{
83+
$this->statisticsEventOutput->writeEvent($this->contentReleaseIdentifier, $this->logPrefix, $event, $additionalPayload);
84+
}
85+
7586
public function withRenderer(RendererIdentifier $rendererIdentifier): self
7687
{
77-
return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $rendererIdentifier);
88+
return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $this->statisticsEventOutput, $rendererIdentifier);
7889
}
7990
}

‎Classes/Core/Infrastructure/RedisStatisticsEventOutput.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
5+
6+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
7+
use Neos\Flow\Annotations as Flow;
8+
9+
class RedisStatisticsEventOutput implements StatisticsEventOutputInterface
10+
{
11+
#[Flow\Inject]
12+
protected RedisStatisticsEventService $redisStatisticsEventService;
13+
14+
public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
15+
{
16+
$this->redisStatisticsEventService->addEvent($contentReleaseIdentifier, $prefix, $event, $additionalPayload);
17+
}
18+
}

‎Classes/Core/Infrastructure/RedisStatisticsEventService.php‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
5+
6+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
7+
use Flowpack\DecoupledContentStore\Core\RedisKeyService;
8+
use Flowpack\DecoupledContentStore\Exception;
9+
use Neos\Flow\Annotations as Flow;
10+
11+
#[Flow\Scope("singleton")]
12+
class RedisStatisticsEventService
13+
{
14+
#[Flow\Inject]
15+
protected RedisClientManager $redisClientManager;
16+
17+
#[Flow\Inject]
18+
protected RedisKeyService $redisKeyService;
19+
20+
public function addEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
21+
{
22+
$this->redisClientManager->getPrimaryRedis()->rPush($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'), json_encode([
23+
'event' => $event,
24+
'prefix' => $prefix,
25+
'additionalPayload' => $additionalPayload,
26+
]));
27+
}
28+
29+
/**
30+
* @param ContentReleaseIdentifier $contentReleaseIdentifier
31+
* @param array<string,string> $where
32+
* @param string[] $groupBy
33+
* @return array<>
34+
* @throws Exception
35+
*/
36+
public function countEvents(
37+
ContentReleaseIdentifier $contentReleaseIdentifier,
38+
array $where,
39+
array $groupBy,
40+
): array
41+
{
42+
$redis = $this->redisClientManager->getPrimaryRedis();
43+
$key = $this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents');
44+
$chunkSize = 1000;
45+
46+
$countedEvents = [];
47+
48+
$listLength = $redis->lLen($key);
49+
for ($start = 0; $start < $listLength; $start += $chunkSize) {
50+
$events = $redis->lRange($key, $start, $start + $chunkSize - 1);
51+
52+
foreach ($events as $eventJson) {
53+
$event = $this->flatten(json_decode($eventJson, true));
54+
if($this->shouldCount($event, $where)) {
55+
$group = $this->groupValues($event, $groupBy);
56+
$eventKey = json_encode($group);
57+
if (array_key_exists($eventKey, $countedEvents)) {
58+
$countedEvents[$eventKey]['count'] += 1;
59+
} else {
60+
$countedEvents[$eventKey] = array_merge(['count' => 1], $group);
61+
}
62+
}
63+
}
64+
}
65+
// throw away the keys and sort in _reverse_ order by count
66+
usort($countedEvents, fn($a, $b) => $b['count'] - $a['count']);
67+
return $countedEvents;
68+
}
69+
70+
/**
71+
* @phpstan-type JSONArray array<string, string|JSONArray>
72+
*´
73+
* @param JSONArray $array
74+
* @return array<string,string>
75+
*/
76+
private function flatten(array $array): array
77+
{
78+
$results = [];
79+
80+
foreach ($array as $key => $value) {
81+
if (is_array($value) && ! empty($value)) {
82+
foreach ($this->flatten($value) as $subKey => $subValue) {
83+
$results[$key . '.' . $subKey] = $subValue;
84+
}
85+
} else {
86+
$results[$key] = $value;
87+
}
88+
}
89+
90+
return $results;
91+
}
92+
93+
/**
94+
* @param array<string,string> $event
95+
* @param array<string,string> $where
96+
* @return bool
97+
*/
98+
private function shouldCount(array $event, array $where): bool
99+
{
100+
foreach ($where as $key=>$value) {
101+
if (!array_key_exists($key, $event) || $event[$key] !== $value) {
102+
return false;
103+
}
104+
}
105+
return true;
106+
}
107+
108+
/**
109+
* @param array<string,string> $event
110+
* @param string[] $groupedBy
111+
* @return array<string,string>
112+
*/
113+
private function groupValues(array $event, array $groupedBy): array
114+
{
115+
$group = [];
116+
foreach ($groupedBy as $path) {
117+
$group[$path] = $event[$path] ?? null;
118+
}
119+
return $group;
120+
}
121+
}

‎Classes/Core/Infrastructure/StatisticsEventOutputInterface.php‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
5+
6+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
7+
8+
interface StatisticsEventOutputInterface
9+
{
10+
public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void;
11+
}

Classes/NodeRendering/Infrastructure/RedisRenderingStatisticsStore.php renamed to Classes/NodeRendering/Infrastructure/RedisRenderingTimeStatisticsStore.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
/**
1515
* @Flow\Scope("singleton")
1616
*/
17-
class RedisRenderingStatisticsStore
17+
class RedisRenderingTimeStatisticsStore
1818
{
1919

2020
/**

‎Classes/NodeRendering/NodeRenderOrchestrator.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
1010
use Flowpack\DecoupledContentStore\NodeRendering\Extensibility\NodeRenderingExtensionManager;
1111
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager;
12-
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore;
12+
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingTimeStatisticsStore;
1313
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\ExitEvent;
1414
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\RenderingIterationCompletedEvent;
1515
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\RenderingQueueFilledEvent;
@@ -70,7 +70,7 @@ class NodeRenderOrchestrator
7070

7171
/**
7272
* @Flow\Inject
73-
* @var RedisRenderingStatisticsStore
73+
* @var RedisRenderingTimeStatisticsStore
7474
*/
7575
protected $redisRenderingStatisticsStore;
7676

‎Configuration/Settings.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,11 @@ Flowpack:
136136
transfer: true
137137
transferMode: 'dump'
138138
isRequired: true
139+
statisticsEvents:
140+
redisKeyPostfix: 'statisticsEvents'
141+
transfer: false
142+
transferMode: 'dump'
143+
isRequired: false
139144

140145
# can be used on the consuming site to ensure non-breaking deployments for changes in the config
141146
configEpoch:

0 commit comments

Comments
 (0)