From 29ce1db5bba81eec8cc16c798d2804d3c22b8be7 Mon Sep 17 00:00:00 2001 From: nasseeder1 Date: Thu, 2 Apr 2020 20:59:26 +0000 Subject: [PATCH] 93_Log2Syslog: Parser UniFi Controller Syslog and Netconsole messages, code review git-svn-id: https://svn.fhem.de/fhem/trunk@21582 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 2 + fhem/FHEM/93_Log2Syslog.pm | 535 ++++++++++++++++++++----------------- 2 files changed, 295 insertions(+), 242 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index 88535624a..b4140f13b 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,7 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_Log2Syslog: Parser UniFi Controller Syslog and Netconsole + messages, code review - feature: 73_AutoShuttersControl: Code Review and change shading logic for night - bugfix: 93_Log2Syslog: fix warning uninitialized value, some code review diff --git a/fhem/FHEM/93_Log2Syslog.pm b/fhem/FHEM/93_Log2Syslog.pm index 62428f6cd..122eb1a7f 100644 --- a/fhem/FHEM/93_Log2Syslog.pm +++ b/fhem/FHEM/93_Log2Syslog.pm @@ -39,6 +39,7 @@ eval "use FHEM::Meta;1" or my $mod # Versions History intern: our %Log2Syslog_vNotesIntern = ( + "5.9.0" => "01.04.2020 Parser UniFi Controller Syslog (BSD Format) and Netconsole messages, more code review (e.g. remove prototypes) ", "5.8.3" => "31.03.2020 fix warning uninitialized value \$pp in pattern match (m//) at line 465, Forum: topic,75426.msg1036553.html#msg1036553, some code review ", "5.8.2" => "28.07.2019 fix warning uninitialized value in numeric ge (>=) at line 662 ", "5.8.1" => "23.07.2019 attribute waitForEOF rename to useEOF, useEOF also for type sender ", @@ -60,8 +61,8 @@ our %Log2Syslog_vNotesIntern = ( "5.2.0" => "02.10.2018 added direct help for attributes", "5.1.0" => "01.10.2018 new get versionNotes command", "5.0.1" => "27.09.2018 Log2Syslog_closesock if write error:.* , delete readings code changed", - "5.0.0" => "26.09.2018 TCP-Server in Collector-mode, HIPCACHE added, PROFILE as Internal, Parse_Err_No as reading, - octetCount attribute, TCP-SSL-support, set 'reopen' command, code fixes", + "5.0.0" => "26.09.2018 TCP-Server in Collector-mode, HIPCACHE added, PROFILE as Internal, Parse_Err_No as reading ". + "octetCount attribute, TCP-SSL-support, set 'reopen' command, code fixes", "4.8.5" => "20.08.2018 BSD/parseFn parsing changed, BSD setpayload changed, new variable \$IGNORE in parseFn", "4.8.4" => "15.08.2018 BSD parsing changed", "4.8.3" => "14.08.2018 BSD setpayload changed, BSD parsing changed, Internal MYFQDN", @@ -72,40 +73,42 @@ our %Log2Syslog_vNotesIntern = ( "4.6.1" => "10.08.2018 some perl warnings, changed IETF Parser", "4.6.0" => "08.08.2018 set sendTestMessage added, Attribute 'contDelimiter', 'respectSeverity'", "4.5.1" => "07.08.2018 BSD Regex changed, setpayload of BSD changed", - "4.5.0" => "06.08.2018 Regex capture groups used in parsePayload to set variables, parsing of BSD changed, - Attribute 'makeMsgEvent' added", + "4.5.0" => "06.08.2018 Regex capture groups used in parsePayload to set variables, parsing of BSD changed ". + "Attribute 'makeMsgEvent' added", "4.4.0" => "04.08.2018 Attribute 'outputFields' added", "4.3.0" => "03.08.2018 Attribute 'parseFn' added", "4.2.0" => "03.08.2018 evaluate sender peer ip-address/hostname, use it as reading in event generation", "4.1.0" => "02.08.2018 state event generation changed", "4.0.0" => "30.07.2018 server mode (Collector)", - "3.2.1" => "04.05.2018 fix compatibility with newer IO::Socket::SSL on debian 9, attr ssldebug for - debugging SSL messages", + "3.2.1" => "04.05.2018 fix compatibility with newer IO::Socket::SSL on debian 9, attr ssldebug for ". + "debugging SSL messages", "3.2.0" => "22.11.2017 add NOTIFYDEV if possible", "3.1.0" => "28.08.2017 get-function added, commandref revised, \$readingFnAttributes deleted", "3.0.0" => "27.08.2017 change attr type to protocol, ready to check in", "2.6.0" => "26.08.2017 more than one Log2Syslog device can be created", - "2.5.2" => "26.08.2018 fix in splitting timestamp, change Log2Syslog_trate using internaltimer with attr - rateCalcRerun, function Log2Syslog_closesock", + "2.5.2" => "26.08.2018 fix in splitting timestamp, change Log2Syslog_trate using internaltimer with attr ". + "rateCalcRerun, function Log2Syslog_closesock", "2.5.1" => "24.08.2017 some fixes", "2.5.0" => "23.08.2017 TLS encryption available, new readings, \$readingFnAttributes", - "2.4.1" => "21.08.2017 changes in sub Log2Syslog_charfilter, change PROCID to \$hash->{SEQNO} - switch to non-blocking in subs event/Log2Syslog_fhemlog", + "2.4.1" => "21.08.2017 changes in Log2Syslog_charfilter, change PROCID to \$hash->{SEQNO} ". + "switch to non-blocking in subs event/Log2Syslog_fhemlog", "2.4.0" => "20.08.2017 new sub Log2Syslog_Log3slog for entries in local fhemlog only -> verbose support", - "2.3.1" => "19.08.2017 commandref revised", + "2.3.1" => "19.08.2017 commandref revised ", "2.3.0" => "18.08.2017 new parameter 'ident' in DEF, sub setidex, Log2Syslog_charfilter", - "2.2.0" => "17.08.2017 set BSD data length, set only acceptable characters (USASCII) in payload, - commandref revised", + "2.2.0" => "17.08.2017 set BSD data length, set only acceptable characters (USASCII) in payload ". + "commandref revised ", "2.1.0" => "17.08.2017 sub Log2Syslog_opensock created", "2.0.0" => "16.08.2017 create syslog without SYS::SYSLOG", - "1.1.1" => "13.08.2017 registrate Log2Syslog_fhemlog to %loginform in case of sending fhem-log - attribute timeout, commandref revised", + "1.1.1" => "13.08.2017 registrate Log2Syslog_fhemlog to %loginform in case of sending fhem-log ". + "attribute timeout, commandref revised", "1.1.0" => "26.07.2017 add regex search to sub Log2Syslog_fhemlog", "1.0.0" => "25.07.2017 initial version" ); # Versions History extern: our %Log2Syslog_vNotesExtern = ( + "5.9.0" => "01.04.2020 The new option \"UniFi\" of attribute \"parseProfil\" provedes a new Parser for UniFi Controller Syslog messages ". + "and Netconsole messages. It was tested with UniFi AP-AC-Lite but should run with all Unifi products. ", "5.8.1" => "23.07.2019 New attribute \"useParsefilter\" to remove other characters than ASCII from payload before parse it. ". "New attribute \"useEOF\" to parse not till the sender was sending an EOF signal (Collector), or in ". "case of model Sender, after transmission an EOF signal is send. A bugfix for ". @@ -246,12 +249,12 @@ my %RFC5425len = ("DL" => 8192, # max. Lange Message insgesamt mit TLS ############################################################################### # Forward declarations # -sub Log2Syslog_Log3slog($$$); +sub Log2Syslog_Log3slog; use vars qw(%Log2Syslog_vHintsExt_en); use vars qw(%Log2Syslog_vHintsExt_de); ############################################################################### -sub Log2Syslog_Initialize($) { +sub Log2Syslog_Initialize { my ($hash) = @_; $hash->{DefFn} = "Log2Syslog_Define"; @@ -272,7 +275,7 @@ sub Log2Syslog_Initialize($) { "logFormat:BSD,IETF ". "makeEvent:no,intern,reading ". "outputFields:sortable-strict,PRIVAL,FAC,SEV,TS,HOST,DATE,TIME,ID,PID,MID,SDFIELD,CONT ". - "parseProfile:Automatic,BSD,IETF,TPLink-Switch,raw,ParseFn ". + "parseProfile:Automatic,BSD,IETF,TPLink-Switch,UniFi,raw,ParseFn ". "parseFn:textField-long ". "respectSeverity:multiple-strict,Emergency,Alert,Critical,Error,Warning,Notice,Informational,Debug ". "octetCount:1,0 ". @@ -294,10 +297,10 @@ return; } ############################################################################### -sub Log2Syslog_Define($@) { +sub Log2Syslog_Define { my ($hash, $def) = @_; - my @a = split("[ \t][ \t]*", $def); - my $name = $hash->{NAME}; + my @a = split("[ \t][ \t]*", $def); + my $name = $hash->{NAME}; return "Error: Perl module ".$MissModulSocket." is missing. Install it on Debian with: sudo apt-get install libio-socket-multicast-perl" if($MissModulSocket); return "Error: Perl module ".$MissModulNDom." is missing." if($MissModulNDom); @@ -318,7 +321,7 @@ sub Log2Syslog_Define($@) { $hash->{MODEL} = "Collector"; $hash->{PROFILE} = "Automatic"; readingsSingleUpdate ($hash, 'Parse_Err_No', 0, 1); # Fehlerzähler für Parse-Errors auf 0 - Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - entering Syslog servermode ..."); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - entering Syslog servermode ..."); Log2Syslog_initServer("$name,global"); } else { # Sendermode @@ -342,7 +345,6 @@ sub Log2Syslog_Define($@) { } $hash->{SEQNO} = 1; # PROCID in IETF, wird kontinuierlich hochgezählt - $logInform{$hash->{NAME}} = "Log2Syslog_fhemlog"; # Funktion die in hash %loginform für $name eingetragen wird $hash->{HELPER}{SSLVER} = "n.a."; # Initialisierung $hash->{HELPER}{SSLALGO} = "n.a."; # Initialisierung @@ -355,11 +357,11 @@ sub Log2Syslog_Define($@) { Log2Syslog_setVersionInfo($hash); readingsBeginUpdate($hash); - readingsBulkUpdate($hash, "SSL_Version", "n.a."); - readingsBulkUpdate($hash, "SSL_Algorithm", "n.a."); - readingsBulkUpdate($hash, "Transfered_logs_per_minute", 0); - readingsBulkUpdate($hash, "state", "initialized") if($hash->{MODEL}=~/Sender/); - readingsEndUpdate($hash,1); + readingsBulkUpdate ($hash, "SSL_Version", "n.a."); + readingsBulkUpdate ($hash, "SSL_Algorithm", "n.a."); + readingsBulkUpdate ($hash, "Transfered_logs_per_minute", 0); + readingsBulkUpdate ($hash, "state", "initialized") if($hash->{MODEL}=~/Sender/); + readingsEndUpdate ($hash,1); Log2Syslog_trate($hash); # regelm. Berechnung Transfer Rate starten @@ -370,8 +372,8 @@ return; # Syslog Collector (Server-Mode) initialisieren # (im Collector Model) ################################################################################################# -sub Log2Syslog_initServer($) { - my ($a) = @_; +sub Log2Syslog_initServer { + my ($a) = @_; my ($name,$global) = split(",",$a); my $hash = $defs{$name}; my $err; @@ -386,7 +388,7 @@ sub Log2Syslog_initServer($) { # Inititialisierung FHEM ist fertig -> Attribute geladen my $port = AttrVal($name, "TLS", 0)?AttrVal($name, "port", 6514):AttrVal($name, "port", 1514); my $protocol = lc(AttrVal($name, "protocol", "udp")); - my $lh = ($global ? ($global eq "global" ? undef : $global) : ($hash->{IPV6} ? "::1" : "127.0.0.1")); + my $lh = ($global ? ($global eq "global" ? undef : $global) : ($hash->{IPV6} ? "::1" : "127.0.0.1")); Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Opening socket on interface \"$global\" ..."); @@ -400,7 +402,7 @@ sub Log2Syslog_initServer($) { ); if(!$hash->{SERVERSOCKET}) { $err = "Can't open Syslog Collector at $port: $!"; - Log2Syslog_Log3slog ($hash, 1, "Log2Syslog $name - $err"); + Log2Syslog_Log3slog ($hash, 1, "Log2Syslog $name - $err"); readingsSingleUpdate ($hash, 'state', $err, 1); return; } @@ -411,7 +413,7 @@ sub Log2Syslog_initServer($) { my $ret = TcpServer_Open($hash,$port,$lh); if($ret) { $err = "Can't open Syslog TCP Collector at $port: $ret"; - Log2Syslog_Log3slog ($hash, 1, "Log2Syslog $name - $err"); + Log2Syslog_Log3slog ($hash, 1, "Log2Syslog $name - $err"); readingsSingleUpdate ($hash, 'state', $err, 1); return; } @@ -422,7 +424,7 @@ sub Log2Syslog_initServer($) { $hash->{HELPER}{OLDSEQNO} = $hash->{SEQNO}; # Init Sequenznummer f. Ratenbestimmung $hash->{INTERFACE} = $lh?$lh:"global"; - Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - port $hash->{PORT}/$protocol opened for Syslog Collector on interface \"$hash->{INTERFACE}\""); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - port $hash->{PORT}/$protocol opened for Syslog Collector on interface \"$hash->{INTERFACE}\""); readingsSingleUpdate ($hash, "state", "initialized", 1); delete($readyfnlist{"$name.$port"}); $selectlist{"$name.$port"} = $hash; @@ -438,9 +440,10 @@ return; # ######################################################################################################## # called from the global loop, when the select for hash->{FD} reports data -sub Log2Syslog_Read($@) { +sub Log2Syslog_Read { my ($hash,$reread) = @_; my $socket = $hash->{SERVERSOCKET}; + my ($err,$sev,$data,$ts,$phost,$pl,$ignore,$st,$len,$mlen,$evt,$pen,$rhash); return if($init_done != 1); @@ -486,7 +489,7 @@ sub Log2Syslog_Read($@) { # Syslog Sätze mit Octet Count -> Transmission of Syslog Messages over TCP https://tools.ietf.org/html/rfc6587 my $i = 0; $mlen = $+{mlen}; - $tail = $+{tail}; + $tail = $+{tail}; $msg = substr($tail,0,$mlen); chomp $msg; push @load, $msg; @@ -496,7 +499,7 @@ sub Log2Syslog_Read($@) { while($tail && $tail =~ /^(?(\d+))\s(?.*)/s) { $i++; $mlen = $+{mlen}; - $tail = $+{tail}; + $tail = $+{tail}; $msg = substr($tail,0,$mlen); chomp $msg; push @load, $msg; @@ -557,7 +560,7 @@ return; # (sSiehe auch "list TYPE=FHEMWEB", bzw. "man -s2 accept") # ############################################################################### -sub Log2Syslog_getifdata($$@) { +sub Log2Syslog_getifdata { my ($hash,$len,$mlen,$reread) = @_; my $name = $hash->{NAME}; my $socket = $hash->{SERVERSOCKET}; @@ -573,8 +576,7 @@ sub Log2Syslog_getifdata($$@) { my ($data,$ret); if(!$reread) { - if($socket && $protocol =~ /udp/) { - # UDP Datagramm empfangen + if($socket && $protocol =~ /udp/) { # UDP Datagramm empfangen Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ######### new Syslog UDP Receive ######### "); Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); @@ -599,7 +601,7 @@ sub Log2Syslog_getifdata($$@) { my $sslver = $nhash->{CD}->get_sslversion(); my $sslalgo = $nhash->{CD}->get_fingerprint(); readingsSingleUpdate($hash, "SSL_Version", $sslver, 1); - readingsSingleUpdate($hash, "SSL_Algorithm", $sslalgo, 1); + readingsSingleUpdate($hash, "SSL_Algorithm", $sslalgo, 1); } return ($st,$data,$hash); } @@ -621,7 +623,7 @@ sub Log2Syslog_getifdata($$@) { my $c = $hash->{CD}; if($c) { $shash->{HELPER}{TCPPADDR} = $hash->{PEER}; - my $buf; + my $buf; my $off = 0; $ret = sysread($c, $buf, $len); # returns undef on error, 0 at end of file and Integer, number of bytes read on success. @@ -695,16 +697,15 @@ return ($st,undef,$hash); # Parsen Payload für Syslog-Server # (im Collector Model) ############################################################################### -sub Log2Syslog_parsePayload($$) { +sub Log2Syslog_parsePayload { my ($hash,$data) = @_; my $name = $hash->{NAME}; my $pp = AttrVal($name, "parseProfile", $hash->{PROFILE}); - my $pr = (AttrVal($name, "protocol", "UDP")); my $severity = ""; my $facility = ""; my @evf = split(",",AttrVal($name, "outputFields", "FAC,SEV,ID,CONT")); # auszugebene Felder im Event/Reading my $ignore = 0; - my ($Mmm,$dd,$delimiter,$day,$ietf,$err,$pl,$tail); + my ($Mmm,$dd,$day,$ietf,$err,$pl,$tail); $data = Log2Syslog_parsefilter($data) if(AttrVal($name,"useParsefilter",0)); # Steuerzeichen werden entfernt (Achtung auch CR/LF) @@ -730,17 +731,15 @@ sub Log2Syslog_parsePayload($$) { DATA => \$data ); - # Sender Host / IP-Adresse ermitteln, $phost wird Reading im Event - my ($phost) = Log2Syslog_evalPeer($hash); + my ($phost) = Log2Syslog_evalPeer($hash); # Sender Host / IP-Adresse ermitteln, $phost wird Reading im Event Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - raw message -> $data"); - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); # Istzeit Ableitung - $year = $year+1900; + my $year = strftime "%Y", localtime; # aktuelles Jahr if($pp =~ /^Automatic/) { - $pp = "raw"; Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - Analyze message format automatically ..."); + $pp = "raw"; $data =~ /^<(?\d{1,3})>(?\w{3}).*$/; $tail = $+{tail}; # Test auf BSD-Format @@ -763,24 +762,23 @@ sub Log2Syslog_parsePayload($$) { } if($pp =~ /raw/) { - # Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - $data"); $ts = TimeNow(); $pl = $data; } elsif ($pp eq "BSD") { # BSD Protokollformat https://tools.ietf.org/html/rfc3164 # Beispiel data "<$prival>$month $day $time $myhost $id: $otp" - $data =~ /^<(?\d{1,3})>(?.*)$/; - $prival = $+{prival}; # must + $data =~ /^<(?\d{1,3})>(?.*)$/; + $prival = $+{prival}; # must $tail = $+{tail}; - $tail =~ /^((?\w{3})\s+(?\d{1,2})\s+(?