From c316c4c8f1bdd9e2ed4994c9a66bd4d344d16c20 Mon Sep 17 00:00:00 2001 From: nasseeder1 Date: Tue, 23 Jul 2019 21:26:25 +0000 Subject: [PATCH] 93_Log2Syslog: V5.8.1, new attributes and further developments regarding to Forum: topic,75426.msg958836.html#msg958836 git-svn-id: https://svn.fhem.de/fhem/trunk@19892 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 3 + fhem/FHEM/93_Log2Syslog.pm | 376 ++++++++++++++++------- fhem/contrib/DS_Starter/93_Log2Syslog.pm | 5 +- 3 files changed, 277 insertions(+), 107 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index 16e5b2d64..edca9b229 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,8 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_Log2Syslog: V5.8.1, new attributes and further developments + regarding to Forum: topic,75426.msg958836.html#msg958836, + use "get versionNotes" for further informations - change: 77_UWZ: remove last InternalTimer Fn parameter - feature: 14_SD_WS.pm: crc calculation fixes and new protocol for Atech wireless weather station diff --git a/fhem/FHEM/93_Log2Syslog.pm b/fhem/FHEM/93_Log2Syslog.pm index 2c38fe9b8..5e365c89e 100644 --- a/fhem/FHEM/93_Log2Syslog.pm +++ b/fhem/FHEM/93_Log2Syslog.pm @@ -41,6 +41,14 @@ eval "use FHEM::Meta;1" or my $modMetaAbsent = 1; # Versions History intern: our %Log2Syslog_vNotesIntern = ( + "5.8.1" => "23.07.2019 attribute waitForEOF rename to useEOF, useEOF also for type sender ", + "5.8.0" => "20.07.2019 attribute waitForEOF, solution for Forum: https://forum.fhem.de/index.php/topic,75426.msg958836.html#msg958836 ", + "5.7.0" => "20.07.2019 change logging and chomp received data, use raw parse format if automatic mode don't detect a valid format, ". + "change getifdata tcp stack error handling (if sysread undef)", + "5.6.5" => "19.07.2019 bugfix parse BSD if ID (TAG) is used, function DbLog_splitFn -> Log2Syslog_DbLogSplit, new attribute useParsefilter ", + "5.6.4" => "19.07.2019 minor changes and fixes (max. lenth read to 16384, code && logging) ", + "5.6.3" => "18.07.2019 fix state reading if changed disabled attribute ", + "5.6.2" => "17.07.2019 Forum: https://forum.fhem.de/index.php/topic,75426.msg958836.html#msg958836 first try", "5.6.1" => "24.03.2019 prevent module from deactivation in case of unavailable Meta.pm ", "5.6.0" => "23.03.2019 attribute exclErrCond to exclude events from rating as \"error\" ", "5.5.0" => "18.03.2019 prepare for Meta.pm ", @@ -98,6 +106,10 @@ our %Log2Syslog_vNotesIntern = ( # Versions History extern: our %Log2Syslog_vNotesExtern = ( + "5.8.1" => "23.07.2019 New attribute \"useParsefilter\" to remove other characters than ASCII from payload before parse it. ". + "New attribute \"useEOF\" to parse not till the sender was sending an EOF signal (Collector), or in ". + "case of model Sender, after transmission an EOF signal is send. A bugfix for ". + "parsing BSD if the ID (TAG) is used was implemented. Minor other fixes and changes. ", "5.6.0" => "23.03.2019 New attribute \"exclErrCond\" to exclude events from rating as \"Error\" even though the ". "event contains the text \"Error\". ", "5.4.0" => "17.03.2019 New feature parseProfile = Automatic. The module may detect the message format BSD or IETF automatically in server mode ", @@ -242,14 +254,15 @@ use vars qw(%Log2Syslog_vHintsExt_de); sub Log2Syslog_Initialize($) { my ($hash) = @_; - $hash->{DefFn} = "Log2Syslog_Define"; - $hash->{UndefFn} = "Log2Syslog_Undef"; - $hash->{DeleteFn} = "Log2Syslog_Delete"; - $hash->{SetFn} = "Log2Syslog_Set"; - $hash->{GetFn} = "Log2Syslog_Get"; - $hash->{AttrFn} = "Log2Syslog_Attr"; - $hash->{NotifyFn} = "Log2Syslog_eventlog"; - $hash->{ReadFn} = "Log2Syslog_Read"; + $hash->{DefFn} = "Log2Syslog_Define"; + $hash->{UndefFn} = "Log2Syslog_Undef"; + $hash->{DeleteFn} = "Log2Syslog_Delete"; + $hash->{SetFn} = "Log2Syslog_Set"; + $hash->{GetFn} = "Log2Syslog_Get"; + $hash->{AttrFn} = "Log2Syslog_Attr"; + $hash->{NotifyFn} = "Log2Syslog_eventlog"; + $hash->{DbLog_splitFn} = "Log2Syslog_DbLogSplit"; + $hash->{ReadFn} = "Log2Syslog_Read"; $hash->{AttrList} = "addStateEvent:1,0 ". "disable:1,0,maintenance ". @@ -263,13 +276,15 @@ sub Log2Syslog_Initialize($) { "parseFn:textField-long ". "respectSeverity:multiple-strict,Emergency,Alert,Critical,Error,Warning,Notice,Informational,Debug ". "octetCount:1,0 ". + "protocol:UDP,TCP ". + "port ". + "rateCalcRerun ". "ssldebug:0,1,2,3 ". "sslCertPrefix ". "TLS:1,0 ". "timeout ". - "protocol:UDP,TCP ". - "port ". - "rateCalcRerun ". + "useParsefilter:0,1 ". + "useEOF:1,0 ". $readingFnAttributes ; @@ -426,23 +441,25 @@ return; sub Log2Syslog_Read($@) { my ($hash,$reread) = @_; my $socket = $hash->{SERVERSOCKET}; - my ($err,$sev,$data,$ts,$phost,$pl,$ignore,$st,$len,$evt,$pen); + my ($err,$sev,$data,$ts,$phost,$pl,$ignore,$st,$len,$mlen,$evt,$pen); return if($init_done != 1); - my $mlen = 8192; # maximale Länge des Syslog-Frames als Begrenzung falls kein EOF - # vom Sender initiiert wird (Endlosschleife vermeiden) + # maximale Länge des Syslog-Frames als Begrenzung falls kein EOF + # vom Sender initiiert wird (Endlosschleife vermeiden) + $mlen = 16384; + $len = 8192; + if($hash->{TEMPORARY}) { # temporäre Instanz angelegt durch TcpServer_Accept - $len = 8192; ($st,$data,$hash) = Log2Syslog_getifdata($hash,$len,$mlen,$reread); } my $name = $hash->{NAME}; return if(IsDisabled($name) || Log2Syslog_IsMemLock($hash)); my $pp = $hash->{PROFILE}; - my $mevt = AttrVal($name, "makeEvent", "intern"); # wie soll Reading/Event erstellt werden - my $sevevt = AttrVal($name, "respectSeverity", ""); # welcher Schweregrad soll berücksichtigt werden (default: alle) + my $mevt = AttrVal($name, "makeEvent", "intern"); # wie soll Reading/Event erstellt werden + my $sevevt = AttrVal($name, "respectSeverity", ""); # welcher Schweregrad soll berücksichtigt werden (default: alle) if($pp =~ /BSD/) { # BSD-Format @@ -452,16 +469,13 @@ sub Log2Syslog_Read($@) { # IETF-Format $len = $RFC5425len{DL}; - } else { - # raw oder User eigenes Format - $len = 8192; } if($socket) { ($st,$data,$hash) = Log2Syslog_getifdata($hash,$len,$mlen,$reread); } - if($data) { + if($data) { # parse Payload my (@load,$mlen,$msg,$tail); if($data =~ /^(?(\d+))\s(?.*)/s) { @@ -488,8 +502,8 @@ sub Log2Syslog_Read($@) { } else { @load = split("[\r\n]",$data); } - - foreach my $line (@load) { + + foreach my $line (@load) { ($err,$ignore,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$line); $hash->{SEQNO}++; if($err) { @@ -500,7 +514,7 @@ sub Log2Syslog_Read($@) { } elsif ($ignore) { Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name -> dataset was ignored by parseFn"); } else { - return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen + return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen $st = "active"; if($mevt =~ /intern/) { # kein Reading, nur Event @@ -543,73 +557,100 @@ return; # ############################################################################### sub Log2Syslog_getifdata($$@) { - my ($hash,$len,$mlen,$reread) = @_; - my $name = $hash->{NAME}; - my $socket = $hash->{SERVERSOCKET}; - my $protocol = lc(AttrVal($name, "protocol", "udp")); + my ($hash,$len,$mlen,$reread) = @_; + my $name = $hash->{NAME}; + my $socket = $hash->{SERVERSOCKET}; + my $protocol = lc(AttrVal($name, "protocol", "udp")); + my ($eof,$buforun) = (0,0); + if($hash->{TEMPORARY}) { # temporäre Instanz abgelegt durch TcpServer_Accept $protocol = "tcp"; } + my $st = ReadingsVal($name,"state","active"); my ($data,$ret); - if($socket && $protocol =~ /udp/) { - # UDP Datagramm empfangen - Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); - Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ######### new Syslog UDP Parsing ######### "); - Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); - unless($socket->recv($data, $len)) { - Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); - $data = '' if(length($data) == 0); - $st = "receive error - see logfile"; - } - } elsif ($protocol =~ /tcp/) { - if($hash->{SERVERSOCKET}) { # Accept and create a child - my $nhash = TcpServer_Accept($hash, "Log2Syslog"); - return ($st,$data,$hash) if(!$nhash); - $nhash->{CD}->blocking(0); - if($nhash->{SSL}) { - my $sslver = $nhash->{CD}->get_sslversion(); - my $sslalgo = $nhash->{CD}->get_fingerprint(); - readingsSingleUpdate($hash, "SSL_Version", $sslver, 1); - readingsSingleUpdate($hash, "SSL_Algorithm", $sslalgo, 1); - } - return ($st,$data,$hash); - } + if(!$reread) { + if($socket && $protocol =~ /udp/) { + # UDP Datagramm empfangen + Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); + Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ######### new Syslog UDP Receive ######### "); + Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ####################################################### "); - my $sname = $hash->{SNAME}; - my $cname = $hash->{NAME}; - my $shash = $defs{$sname}; # Hash des Log2Syslog-Devices bei temporärer TCP-Serverinstanz + unless($socket->recv($data, $len)) { + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); + $data = '' if(length($data) == 0); + $st = "receive error - see logfile"; + } else { + my $dl = length($data); + chomp $data; + Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - Buffer ".$dl." chars ready to parse:\n$data"); + } + return ($st,$data,$hash); - Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ####################################################### "); - Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ######### new Syslog TCP Parsing ######### "); - Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ####################################################### "); - Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - childname: $cname"); - $st = ReadingsVal($sname,"state","active"); - my $c = $hash->{CD}; - if($c) { - $shash->{HELPER}{TCPPADDR} = $hash->{PEER}; - if(!$reread) { + } elsif ($protocol =~ /tcp/) { + if($hash->{SERVERSOCKET}) { # Accept and create a child + my $nhash = TcpServer_Accept($hash, "Log2Syslog"); + return ($st,$data,$hash) if(!$nhash); + $nhash->{CD}->blocking(0); + if($nhash->{SSL}) { + my $sslver = $nhash->{CD}->get_sslversion(); + my $sslalgo = $nhash->{CD}->get_fingerprint(); + readingsSingleUpdate($hash, "SSL_Version", $sslver, 1); + readingsSingleUpdate($hash, "SSL_Algorithm", $sslalgo, 1); + } + return ($st,$data,$hash); + } + + # Child, $hash ist Hash der temporären Instanz + my $sname = $hash->{SNAME}; + my $cname = $hash->{NAME}; + my $shash = $defs{$sname}; # Hash des Log2Syslog-Devices bei temporärer TCP-Serverinstanz + my $uef = AttrVal($sname, "useEOF", 0); + my $tlsv = ReadingsVal($sname,"SSL_Version",''); + + Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ####################################################### "); + Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ######### new Syslog TCP Receive ######### "); + Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - ####################################################### "); + Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - await EOF: $uef, SSL: $tlsv"); + Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - childname: $cname"); + + $st = ReadingsVal($sname,"state","active"); + my $c = $hash->{CD}; + if($c) { + $shash->{HELPER}{TCPPADDR} = $hash->{PEER}; my $buf; my $off = 0; $ret = sysread($c, $buf, $len); # returns undef on error, 0 at end of file and Integer, number of bytes read on success. - if(!defined($ret) && $! == EWOULDBLOCK ){ - # error + if(!defined($ret) && $! == EWOULDBLOCK){ + # error $hash->{wantWrite} = 1 if(TcpServer_WantWrite($hash)); $hash = $shash; + Log2Syslog_Log3slog ($hash, 2, "Log2Syslog $sname - ERROR - TCP stack error: $!"); return ($st,undef,$hash); } elsif (!$ret) { - # end of file - CommandDelete(undef, $cname); - $hash = $shash; + # EOF or error Log2Syslog_Log3slog ($shash, 4, "Log2Syslog $sname - Connection closed for $cname: ".(defined($ret) ? 'EOF' : $!)); - return ($st,undef,$hash); - + if(!defined($ret)) { + # error + CommandDelete(undef, $cname); + $hash = $shash; + return ($st,undef,$hash); + } else { + # EOF + $eof = 1; + $data = $hash->{BUF}; + CommandDelete(undef, $cname); + } + } + + if(!$eof) { + $hash->{BUF} .= $buf; + Log2Syslog_Log3slog ($shash, 5, "Log2Syslog $sname - Add $ret chars to buffer:\n$buf") if($uef && !$hash->{SSL}); } - $hash->{BUF} .= $buf; if($hash->{SSL} && $c->can('pending')) { while($c->pending()) { @@ -618,19 +659,38 @@ sub Log2Syslog_getifdata($$@) { } } - $data = $hash->{BUF}; - delete $hash->{BUF}; - $hash = $shash; - Log2Syslog_Log3slog ($shash, 5, "Log2Syslog $sname - Buffer content:\n$data"); + $buforun = (length($hash->{BUF}) >= $mlen)?1:0; + + if(!$uef || $hash->{SSL} || $buforun) { + $data = $hash->{BUF}; + delete $hash->{BUF}; + $hash = $shash; + if($data) { + my $dl = length($data); + chomp $data; + Log2Syslog_Log3slog ($shash, 2, "Log2Syslog $sname - WARNING - Buffer overrun ! Enforce parse data.") if($buforun); + Log2Syslog_Log3slog ($shash, 5, "Log2Syslog $sname - Buffer $dl chars ready to parse:\n$data"); + } + return ($st,$data,$hash); + } else { + if($eof) { + $hash = $shash; + my $dl = length($data); + chomp $data; + Log2Syslog_Log3slog ($shash, 5, "Log2Syslog $sname - Buffer $dl chars after EOF ready to parse:\n$data") if($data); + return ($st,$data,$hash); + } + } } + + } else { + $st = "error - no socket opened"; + $data = ''; + return ($st,$data,$hash); } - - } else { - $st = "error - no socket opened"; - $data = ''; } -return ($st,$data,$hash); +return ($st,undef,$hash); } ############################################################################### @@ -641,12 +701,18 @@ sub Log2Syslog_parsePayload($$) { my ($hash,$data) = @_; my $name = $hash->{NAME}; my $pp = AttrVal($name, "parseProfile", $hash->{PROFILE}); + my $pr = (AttrVal($name, "protocol", "UDP")); my $severity = ""; my $facility = ""; my @evf = split(",",AttrVal($name, "outputFields", "FAC,SEV,ID,CONT")); # auszugebene Felder im Event/Reading my $ignore = 0; my ($Mmm,$dd,$delimiter,$day,$ietf,$err,$pl,$tail); + $data = Log2Syslog_parsefilter($data) if(AttrVal($name,"useParsefilter",0)); # Steuerzeichen werden entfernt (Achtung auch CR/LF) + + Log2Syslog_Log3slog ($hash, 4, "Log2Syslog $name - ######### Parse Message ######### "); + Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - parse profile: $pp"); + # Hash zur Umwandlung Felder in deren Variablen my ($prival,$ts,$host,$date,$time,$id,$pid,$mid,$sdfield,$cont); my $fac = ""; @@ -675,7 +741,7 @@ sub Log2Syslog_parsePayload($$) { $year = $year+1900; if($pp =~ /^Automatic/) { - $pp = "unknown"; + $pp = "raw"; Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - Analyze message format automatically ..."); $data =~ /^<(?\d{1,3})>(?\w{3}).*$/; $tail = $+{tail}; @@ -690,12 +756,16 @@ sub Log2Syslog_parsePayload($$) { $time = $+{time}; # must $pp = "IETF" if($prival && $date && $time); } - $hash->{PROFILE} = "Automatic - detected format: $pp"; - Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - Message format \"$pp\" detected. Try Parsing ... ") if($pp ne "unknown"); + if($pp ne "raw") { + $hash->{PROFILE} = "Automatic - detected format: $pp"; + Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - Message format \"$pp\" detected. Try Parsing ... "); + } else { + Log2Syslog_Log3slog($name, 2, "Log2Syslog $name - WARNING - no message format is detected, \"raw\" is used instead. You can specify the correct profile by attribute \"parseProfile\" !"); + } } if($pp =~ /raw/) { - Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - $data"); + # Log2Syslog_Log3slog($name, 4, "Log2Syslog $name - $data"); $ts = TimeNow(); $pl = $data; @@ -703,12 +773,12 @@ sub Log2Syslog_parsePayload($$) { # BSD Protokollformat https://tools.ietf.org/html/rfc3164 # Beispiel data "<$prival>$month $day $time $myhost $id: $otp" $data =~ /^<(?\d{1,3})>(?.*)$/; - $prival = $+{prival}; # must + $prival = $+{prival}; # must $tail = $+{tail}; $tail =~ /^((?\w{3})\s+(?\d{1,2})\s+(?