From 22f6570af5b16289d2d285783b74e59c1570d005 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Dec 2025 07:55:23 +0100 Subject: [PATCH] logwatch extension+review --- contrib/logwatch/radicale | 228 +++++++++++++++++++++++++++----------- 1 file changed, 166 insertions(+), 62 deletions(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 8acbe0ae..c659f62f 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -3,19 +3,32 @@ # Copyright © 2024-2025 Peter Bieringer # # Detail levels -# >= 5: Logins -# >= 10: ResponseTimes +# < 5 : Request + ResponseCounters +# >= 5 : incl. Logins +# >= 10: incl. ResponseTimes + ResponseSize +# >= 15: incl. ResponseTimes + ResponseSize incl. RequestFlags +# >= 18: incl. UserAgents +# >= 20: incl. Locations where supported, anonymize logins + +use Digest::SHA; $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0; +my %ResponseTimesLocUsr; +my %ResponseSizesLocUsr; my %ResponseTimes; my %ResponseSizes; my %Responses; my %Requests; +my %UserAgents; my %Logins; my %Loglevel; my %OtherEvents; +my %Locations; +my %LocationsFile; +my %LoginsHash; + my $sum; my $length; @@ -108,6 +121,36 @@ sub ConvertTokens($) { return $result; } +sub ConvertLoc($) { + my $loc = $_[0]; + if (defined $Locations{$loc}) { + # from cache + return ":L=" . $Locations{$loc}; + } elsif (defined $LocationsFile{$loc}) { + # from cache + return ":L=" . $LocationsFile{$loc}; + } + + if ($loc =~ /\/'$/o) { + $Locations{$loc} = "L=" . substr(Digest::SHA::sha256_hex($loc), 0, 8); + return ":" . $Locations{$loc}; + } else { + $LocationsFile{$loc} = "L="; + return ":" . $Locations{$loc}; + } +} + +sub ConvertLogin($) { + my $login = $_[0]; + if (defined $LoginsHash{$loginc}) { + # from cache + return $LoginsHash{$login}; + } + + $LoginsHash{$login} = "U=" . substr(Digest::SHA::sha256_hex($login), 0, 8); + return $LoginsHash{$login}; +} + while (defined($ThisLine = )) { # count loglevel if ( $ThisLine =~ /\[(DEBUG|INFO|WARNING|ERROR|CRITICAL)\] /o ) { @@ -123,42 +166,60 @@ while (defined($ThisLine = )) { } elsif ( $ThisLine =~ / (\S+) response status/o ) { my $req = $1; - if ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds: (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $3; + if ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds: (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $4; + $req .= ConvertLoc($1) if ($Detail >= 20); ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds: (\d+)/o ) { - $req .= ":R=" . $2; + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds: (\d+)/o ) { + $req .= ":R=" . $3; + $req .= ConvertLoc($1) if ($Detail >= 20); ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $5; - ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { - $req .= ":R=" . $4; - ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $6; - $reqWithFlags = $req . ConvertTokens($5); - ResponseTimesMinMaxSum($reqWithFlags, $2) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + } elsif ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $6; + $req .= ConvertLoc($1) if ($Detail >= 20); + ResponseTimesMinMaxSum($req, $3) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $4, $5) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { $req .= ":R=" . $5; - $reqWithFlags = $req . ConvertTokens($4); - ResponseTimesMinMaxSum($reqWithFlags, $1) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); + $req .= ConvertLoc($1) if ($Detail >= 20); + ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $7; + $req .= ConvertLoc($1) if ($Detail >= 20); + $req .= ConvertTokens($6) if ($Detail >= 15); + ResponseTimesMinMaxSum($req, $3) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $4, $5) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":R=" . $6; + $req .= ConvertLoc($1) if ($Detail >= 20); + $req .= ConvertTokens($6) if ($Detail >= 15); + ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); } $Responses{$req}++; } - elsif ( $ThisLine =~ / (\S+) request for/o ) { + elsif ( $ThisLine =~ / (\S+) request for ('[^']+')/o ) { my $req = $1; - if ( $ThisLine =~ / \S+ request for .* with depth '(\d)' received/o ) { + my $loc = $2; + if ( $ThisLine =~ / with depth '(\d)' received/o ) { $req .= ":D=" . $1; } + $req .= ConvertLoc($loc) if ($Detail >= 20); $Requests{$req}++; + + if ( $ThisLine =~ /using ('.*')/o ) { + my $ua = $1; + # remove unexpected chars + $ua =~ s/[\x00-\x1F\x7F-\xFF]//g; + $ua .= ConvertLoc($loc) if ($Detail >= 20); + $UserAgents{$ua}++ if ($Detail >= 18); + } } elsif ( $ThisLine =~ / (Successful login): '([^']+)'/o ) { - $Logins{$2}++ if ($Detail >= 5); + my $login = $2; + $login = ConvertLogin($login) if ($Detail >= 20); + $Logins{$login}++ if ($Detail >= 5); $OtherEvents{$1}++; } elsif ( $ThisLine =~ / (Failed login attempt) /o ) { @@ -207,57 +268,76 @@ if ($Started) { if (keys %Loglevel) { $sum = Sum(\%Loglevel); print "\n**Loglevel counters**\n"; - printf "%-18s | %7s | %5s |\n", "Loglevel", "cnt", "ratio"; - print "-" x38 . "\n"; + printf "%-18s | %7s | %9s |\n", "Loglevel", "cnt", "ratio"; + print "-" x42 . "\n"; foreach my $level (sort keys %Loglevel) { - printf "%-18s | %7d | %3d%% |\n", $level, $Loglevel{$level}, int(($Loglevel{$level} * 100) / $sum); + printf "%-18s | %7d | %7.3f%% |\n", $level, $Loglevel{$level}, (($Loglevel{$level} * 100) / $sum); } - print "-" x38 . "\n"; - printf "%-18s | %7d | %3d%% |\n", "", $sum, 100; -} - -if (keys %Requests) { - $sum = Sum(\%Requests); - print "\n**Request counters (D=)**\n"; - printf "%-18s | %7s | %5s |\n", "Request", "cnt", "ratio"; - print "-" x38 . "\n"; - foreach my $req (sort keys %Requests) { - printf "%-18s | %7d | %3d%% |\n", $req, $Requests{$req}, int(($Requests{$req} * 100) / $sum); - } - print "-" x38 . "\n"; - printf "%-18s | %7d | %3d%% |\n", "", $sum, 100; -} - -if (keys %Responses) { - $sum = Sum(\%Responses); - $length = MaxLength(\%Responses); - print "\n**Response result counters ((D= R=)**\n"; - printf "%-" . $length . "s | %7s | %5s |\n", "Response", "cnt", "ratio"; - print "-" x($length + 20) . "\n"; - foreach my $req (sort keys %Responses) { - printf "%-" . $length . "s | %7d | %3d%% |\n", $req, $Responses{$req}, int(($Responses{$req} * 100) / $sum); - } - print "-" x($length + 20) . "\n"; - printf "%-" . $length . "s | %7d | %3d%% |\n", "", $sum, 100; + print "-" x42 . "\n"; + printf "%-18s | %7d | %7.3f%% |\n", "", $sum, 100; } if (keys %Logins) { $sum = Sum(\%Logins); $length = MaxLength(\%Logins); print "\n**Successful login counters**\n"; - printf "%-" . $length . "s | %7s | %5s |\n", "Login", "cnt", "ratio"; - print "-" x($length + 20) . "\n"; + printf "%-" . $length . "s | %7s | %9s |\n", "Login", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; foreach my $login (sort keys %Logins) { - printf "%-" . $length . "s | %7d | %3d%% |\n", $login, $Logins{$login}, int(($Logins{$login} * 100) / $sum); + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $login, $Logins{$login}, (($Logins{$login} * 100) / $sum); } - print "-" x($length + 20) . "\n"; - printf "%-" . $length . "s | %7d | %3d%% |\n", "", $sum, 100; + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3d%% |\n", "", $sum, 100; +} + +if (keys %UserAgents) { + $sum = Sum(\%UserAgents); + $length = MaxLength(\%UserAgents); + print "\n**UserAgent Counters**\n"; + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "UserAgent", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $ua (sort keys %UserAgents) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $ua, $UserAgents{$ua}, (($UserAgents{$ua} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3d%% |\n", "", $sum, 100; +} + +if (keys %Requests) { + $sum = Sum(\%Requests); + $length = MaxLength(\%Requests); + print "\n**Request counters (D=)**\n"; + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "Request", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $req (sort keys %Requests) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $req, $Requests{$req}, (($Requests{$req} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-18s | %7d | %7.3f%% |\n", "", $sum, 100; +} + +if (keys %Responses) { + $sum = Sum(\%Responses); + $length = MaxLength(\%Responses); + print "\n**Response result counters ((D= R=)**\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "Response", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $req (sort keys %Responses) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $req, $Responses{$req}, (($Responses{$req} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3f%% |\n", "", $sum, 100; } if (keys %ResponseTimes) { $length = MaxLength(\%ResponseTimes); print "\n**Response timings (counts, seconds) (D= R= F=)**\n"; - print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); printf "%-" . $length . "s | %7s | %7s | %7s | %7s |\n", "Response", "cnt", "min", "max", "avg"; print "-" x($length + 42) . "\n"; foreach my $req (sort keys %ResponseTimes) { @@ -274,6 +354,8 @@ if (keys %ResponseSizes) { for my $type (sort keys %ResponseSizes) { $length = MaxLength($ResponseSizes{$type}); print "\n**Response sizes (counts, bytes: $type) (D= R=)**\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); printf "%-" . $length . "s | %7s | %9s | %9s | %9s |\n", "Response", "cnt", "min", "max", "avg"; print "-" x($length + 48) . "\n"; foreach my $req (sort keys %{$ResponseSizes{$type}}) { @@ -301,6 +383,28 @@ if (keys %OtherList) { } } +if (scalar(keys %LoginsHash) > 0) { + print "\n**Map of login hashes (REMOVE THIS FOR PRIVACY REASONS before submit)**\n"; + $length = MaxLength(\%LoginsHash); + printf "%-10s | %-" . $length . "s | \n", "Hash", "Login"; + print "-" x($length + 15) . "\n"; + foreach my $login (sort { $LoginsHash{$a} cmp $LoginsHash{$b} } keys %LoginsHash) { + printf "%10s | %-" . $length . "s |\n", $LoginsHash{$login}, $login; + } + print "-" x($length + 15) . "\n"; +} + +if (scalar(keys %Locations) > 0) { + print "\n**Map of location hashes (REMOVE THIS FOR PRIVACY REASONS before submit)**\n"; + $length = MaxLength(\%Locations); + printf "%-10s | %-" . $length . "s | \n", "Hash", "Location"; + print "-" x($length + 15) . "\n"; + foreach my $loc (sort { $Locations{$a} cmp $Locations{$b} } keys %Locations) { + printf "%10s | %-" . $length . "s |\n", $Locations{$loc}, $loc; + } + print "-" x($length + 15) . "\n"; +} + exit(0); # vim: shiftwidth=3 tabstop=3 syntax=perl et smartindent