diff --git a/fhem/CHANGED b/fhem/CHANGED index 2f22b3449..6290a4174 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,7 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_Log2Syslog: V4.8.5, new Syslog-Server Mode and some other + improvements, version is moved from contrib - bugfix: 73_GardenaSmartBridge fix get humidity bug, add rename Fn - change: 71_YAMAHA_AVR: renamed attributes: "request-timeout" => "requestTimeout" diff --git a/fhem/FHEM/93_Log2Syslog.pm b/fhem/FHEM/93_Log2Syslog.pm index 32f3c62a9..51b460185 100644 --- a/fhem/FHEM/93_Log2Syslog.pm +++ b/fhem/FHEM/93_Log2Syslog.pm @@ -3,7 +3,7 @@ ###################################################################################################################### # 93_Log2Syslog.pm # -# (c) 2017 by Heiko Maaz +# (c) 2017-2018 by Heiko Maaz # e-mail: Heiko dot Maaz at t-online dot de # # This script is part of fhem. @@ -30,6 +30,23 @@ ###################################################################################################################### # Versions History: # +# 4.8.5 20.08.2018 BSD/parseFn parsing changed, BSD setpayload changed, new variable $IGNORE in parseFn +# 4.8.4 15.08.2018 BSD parsing changed +# 4.8.3 14.08.2018 BSD setpayload changed, BSD parsing changed, Internal MYFQDN +# 4.8.2 13.08.2018 rename makeMsgEvent to makeEvent +# 4.8.1 12.08.2018 IETF-Syslog without VERSION changed, Log verbose 1 to 2 changed in parsePayload +# 4.8.0 12.08.2018 enhanced IETF Parser to match logs without version +# 4.7.0 10.08.2018 Parser for TPLink +# 4.6.1 10.08.2018 some perl warnings, changed IETF Parser +# 4.6.0 08.08.2018 set sendTestMessage added, Attribute "contDelimiter", "respectSeverity" +# 4.5.1 07.08.2018 BSD Regex changed, setpayload of BSD changed +# 4.5.0 06.08.2018 Regex capture groups used in parsePayload to set variables, parsing of BSD changed, +# Attribute "makeMsgEvent" added +# 4.4.0 04.08.2018 Attribute "outputFields" added +# 4.3.0 03.08.2018 Attribute "parseFn" added +# 4.2.0 03.08.2018 evaluate sender peer ip-address/hostname, use it as reading in event generation +# 4.1.0 02.08.2018 state event generation changed +# 4.0.0 30.07.2018 server mode (Collector) # 3.2.1 04.05.2018 fix compatibility with newer IO::Socket::SSL on debian 9, attr ssldebug for # debugging SSL messages # 3.2.0 22.11.2017 add NOTIFYDEV if possible @@ -47,7 +64,7 @@ # 2.3.0 18.08.2017 new parameter "ident" in DEF, sub setidex, Log2Syslog_charfilter # 2.2.0 17.08.2017 set BSD data length, set only acceptable characters (USASCII) in payload # commandref revised -# 2.1.0 17.08.2017 sub Log2Syslog_setsock created +# 2.1.0 17.08.2017 sub Log2Syslog_opensock created # 2.0.0 16.08.2017 create syslog without SYS::SYSLOG # 1.1.1 13.08.2017 registrate Log2Syslog_fhemlog to %loginform in case of sending fhem-log # attribute timeout, commandref revised @@ -58,6 +75,9 @@ package main; use strict; use warnings; +use Scalar::Util qw(looks_like_number); +use Encode qw(encode_utf8); +use Net::Domain qw(hostname hostfqdn hostdomain); eval "use IO::Socket::INET;1" or my $MissModulSocket = "IO::Socket::INET"; eval "use Net::Domain qw(hostname hostfqdn hostdomain domainname);1" or my $MissModulNDom = "Net::Domain"; @@ -66,10 +86,10 @@ eval "use Net::Domain qw(hostname hostfqdn hostdomain domainname);1" or my $Mis # sub Log2Syslog_Log3slog($$$); -my $Log2SyslogVn = "3.2.1"; +my $Log2SyslogVn = "4.8.5"; # Mappinghash BSD-Formatierung Monat -my %Log2Syslog_BSDMonth = ( +our %Log2Syslog_BSDMonth = ( "01" => "Jan", "02" => "Feb", "03" => "Mar", @@ -81,16 +101,80 @@ my %Log2Syslog_BSDMonth = ( "09" => "Sep", "10" => "Oct", "11" => "Nov", - "12" => "Dec" + "12" => "Dec", + "Jan" => "01", + "Feb" => "02", + "Mar" => "03", + "Apr" => "04", + "May" => "05", + "Jun" => "06", + "Jul" => "07", + "Aug" => "08", + "Sep" => "09", + "Oct" => "10", + "Nov" => "11", + "Dec" => "12" ); +# Mappinghash Severity +my %Log2Syslog_Severity = ( + "0" => "Emergency", + "1" => "Alert", + "2" => "Critical", + "3" => "Error", + "4" => "Warning", + "5" => "Notice", + "6" => "Informational", + "7" => "Debug", + "Emergency" => "0", + "Alert" => "1", + "Critical" => "2", + "Error" => "3", + "Warning" => "4", + "Notice" => "5", + "Informational" => "6", + "Debug" => "7" +); + +# Mappinghash Facility +my %Log2Syslog_Facility = ( + "0" => "kernel", + "1" => "user", + "2" => "mail", + "3" => "system", + "4" => "security", + "5" => "syslog", + "6" => "printer", + "7" => "network", + "8" => "UUCP", + "9" => "clock", + "10" => "security", + "11" => "FTP", + "12" => "NTP", + "13" => "log_audit", + "14" => "log_alert", + "15" => "clock", + "16" => "local0", + "17" => "local1", + "18" => "local2", + "19" => "local3", + "20" => "local4", + "21" => "local5", + "22" => "local6", + "23" => "local7" + ); + # Längenvorgaben nach RFC3164 my %RFC3164len = ("TAG" => 32, # max. Länge TAG-Feld "DL" => 1024 # max. Lange Message insgesamt ); # Längenvorgaben nach RFC5425 -my %RFC5425len = ("DL" => 8192 # max. Lange Message insgesamt mit TLS +my %RFC5425len = ("DL" => 8192, # max. Lange Message insgesamt mit TLS + "HST" => 255, # max. Länge Hostname + "ID" => 48, # max. Länge APP-NAME bzw. Ident + "PID" => 128, # max. Länge Proc-ID + "MID" => 32 # max. Länge MSGID ); ############################################################################### @@ -100,20 +184,29 @@ sub Log2Syslog_Initialize($) { $hash->{DefFn} = "Log2Syslog_Define"; $hash->{UndefFn} = "Log2Syslog_Undef"; $hash->{DeleteFn} = "Log2Syslog_Delete"; - $hash->{GetFn} = "Log2Syslog_Get"; + $hash->{SetFn} = "Log2Syslog_Set"; + $hash->{GetFn} = "Log2Syslog_Get"; $hash->{AttrFn} = "Log2Syslog_Attr"; $hash->{NotifyFn} = "Log2Syslog_eventlog"; + $hash->{ReadFn} = "Log2Syslog_Read"; $hash->{AttrList} = "addStateEvent:1,0 ". - "disable:1,0 ". + "disable:1,0,maintenance ". "addTimestamp:0,1 ". + "contDelimiter ". "logFormat:BSD,IETF ". + "makeEvent:no,intern,reading ". + "outputFields:sortable-strict,PRIVAL,FAC,SEV,TS,HOST,DATE,TIME,ID,PID,MID,SDFIELD,CONT ". + "parseProfile:BSD,IETF,TPLink-Switch,raw,ParseFn ". + "parseFn:textField-long ". + "respectSeverity:multiple-strict,Emergency,Alert,Critical,Error,Warning,Notice,Informational,Debug ". "ssldebug:0,1,2,3 ". "TLS:1,0 ". "timeout ". "protocol:UDP,TCP ". "port ". - "rateCalcRerun " + "rateCalcRerun ". + $readingFnAttributes ; return undef; } @@ -122,34 +215,47 @@ return undef; sub Log2Syslog_Define($@) { my ($hash, $def) = @_; my @a = split("[ \t][ \t]*", $def); + my $name = $hash->{NAME}; return "Error: Perl module ".$MissModulSocket." is missing. Install it on Debian with: sudo apt-get install libio-socket-multicast-perl" if($MissModulSocket); return "Error: Perl module ".$MissModulNDom." is missing." if($MissModulNDom); - # Example: define splunklog Log2Syslog splunk.myds.me ident event:.* fhem:.* - return "wrong syntax, use: define Log2Syslog [ident:] [event:] [fhem:] " - if(int(@a)-3 < 0); - + # Example Sender: define splunklog Log2Syslog splunk.myds.me ident:Prod event:.* fhem:.* + # Example Collector: define SyslogServer Log2Syslog + delete($hash->{HELPER}{EVNTLOG}); delete($hash->{HELPER}{FHEMLOG}); delete($hash->{HELPER}{IDENT}); - Log2Syslog_setidrex($hash,$a[3]) if($a[3]); - Log2Syslog_setidrex($hash,$a[4]) if($a[4]); - Log2Syslog_setidrex($hash,$a[5]) if($a[5]); + $hash->{MYFQDN} = hostfqdn(); # MYFQDN eigener Host (f. IETF) + $hash->{MYHOST} = hostname(); # eigener Host (lt. RFC nur Hostname f. BSD) - return "Bad regexp: starting with *" - if((defined($hash->{HELPER}{EVNTLOG}) && $hash->{HELPER}{EVNTLOG} =~ m/^\*/) || (defined($hash->{HELPER}{FHEMLOG}) && $hash->{HELPER}{FHEMLOG} =~ m/^\*/)); - eval { "Hallo" =~ m/^$hash->{HELPER}{EVNTLOG}$/ } if($hash->{HELPER}{EVNTLOG}); - return "Bad regexp: $@" if($@); - eval { "Hallo" =~ m/^$hash->{HELPER}{FHEMLOG}$/ } if($hash->{HELPER}{FHEMLOG}); - return "Bad regexp: $@" if($@); + if(int(@a)-3 < 0){ + # Einrichtung Servermode (Collector) + Log3 ($name, 3, "Log2Syslog $name - entering Syslog servermode ..."); + $hash->{MODEL} = "Collector"; + Log2Syslog_initServer("$name,global"); + } else { + # Sendermode + $hash->{MODEL} = "Sender"; + Log2Syslog_setidrex($hash,$a[3]) if($a[3]); + Log2Syslog_setidrex($hash,$a[4]) if($a[4]); + Log2Syslog_setidrex($hash,$a[5]) if($a[5]); + + eval { "Hallo" =~ m/^$hash->{HELPER}{EVNTLOG}$/ } if($hash->{HELPER}{EVNTLOG}); + return "Bad regexp: $@" if($@); + eval { "Hallo" =~ m/^$hash->{HELPER}{FHEMLOG}$/ } if($hash->{HELPER}{FHEMLOG}); + return "Bad regexp: $@" if($@); - # nur Events dieser Devices an NotifyFn weiterleiten, NOTIFYDEV wird gesetzt wenn möglich - notifyRegexpChanged($hash, $hash->{HELPER}{EVNTLOG}) if($hash->{HELPER}{EVNTLOG}); + return "Bad regexp: starting with *" + if((defined($hash->{HELPER}{EVNTLOG}) && $hash->{HELPER}{EVNTLOG} =~ m/^\*/) || (defined($hash->{HELPER}{FHEMLOG}) && $hash->{HELPER}{FHEMLOG} =~ m/^\*/)); + + # nur Events dieser Devices an NotifyFn weiterleiten, NOTIFYDEV wird gesetzt wenn möglich + notifyRegexpChanged($hash, $hash->{HELPER}{EVNTLOG}) if($hash->{HELPER}{EVNTLOG}); - $hash->{PEERHOST} = $a[2]; # Destination Host (Syslog Server) - $hash->{MYHOST} = hostfqdn(); # FQDN eigener Host + $hash->{PEERHOST} = $a[2]; # Destination Host (Syslog Server) + } + $hash->{SEQNO} = 1; # PROCID in IETF, wird kontinuierlich hochgezählt $hash->{VERSION} = $Log2SyslogVn; $logInform{$hash->{NAME}} = "Log2Syslog_fhemlog"; # Funktion die in hash %loginform für $name eingetragen wird @@ -157,32 +263,628 @@ sub Log2Syslog_Define($@) { $hash->{HELPER}{SSLALGO} = "n.a."; # Initialisierung $hash->{HELPER}{LTIME} = time(); # Init Timestmp f. Ratenbestimmung $hash->{HELPER}{OLDSEQNO} = $hash->{SEQNO}; # Init Sequenznummer f. Ratenbestimmung + $hash->{HELPER}{OLDSTATE} = "initialized"; readingsBeginUpdate($hash); readingsBulkUpdate($hash, "SSL_Version", "n.a."); readingsBulkUpdate($hash, "SSL_Algorithm", "n.a."); readingsBulkUpdate($hash, "Transfered_logs_per_minute", 0); - readingsBulkUpdate($hash, "state", "initialized"); + readingsBulkUpdate($hash, "state", "initialized") if($hash->{MODEL}=~/Sender/); readingsEndUpdate($hash,1); Log2Syslog_trate($hash); # regelm. Berechnung Transfer Rate starten + +return undef; +} + +################################################################################################# +# Syslog Collector (Server-Mode) initialisieren +# (im Collector Model) +################################################################################################# +sub Log2Syslog_initServer($) { + my ($a) = @_; + my ($name,$global) = split(",",$a); + my $hash = $defs{$name}; + + RemoveInternalTimer($hash, "Log2Syslog_initServer"); + return if(IsDisabled($name)); + + if($init_done != 1) { + InternalTimer(gettimeofday()+5, "Log2Syslog_initServer", "$name,$global", 0); + return; + } + # Inititialisierung FHEM ist fertig -> Attribute geladen + my $port = AttrVal($name, "TLS", 0)?AttrVal($name, "port", 6514):AttrVal($name, "port", 1514); + my $protocol = lc(AttrVal($name, "protocol", "udp")); + my $lh = $global ? ($global eq "global"? undef : $global) : ($hash->{IPV6} ? "::1" : "127.0.0.1"); + + Log3 $hash, 3, "Log2Syslog $name - Opening socket ..."; + + $hash->{SERVERSOCKET} = IO::Socket::INET->new( + Domain => ($hash->{IPV6} ? AF_INET6() : AF_UNSPEC), # Linux bug + LocalHost => $lh, + Proto => $protocol, + LocalPort => $port, + ReuseAddr => 1 + ); + if(!$hash->{SERVERSOCKET}) { + my $err = "Can't open Syslog Collector at $port: $!"; + Log3 ($hash, 1, "Log2Syslog $name - $err"); + readingsSingleUpdate ($hash, 'state', $err, 1); + return; + } + + $hash->{FD} = $hash->{SERVERSOCKET}->fileno(); + $hash->{PORT} = $hash->{SERVERSOCKET}->sockport(); + $hash->{PROTOCOL} = $protocol; + $hash->{SEQNO} = 1; # PROCID wird kontinuierlich pro empfangenen Datensatz hochgezählt + $hash->{HELPER}{OLDSEQNO} = $hash->{SEQNO}; # Init Sequenznummer f. Ratenbestimmung + $hash->{INTERFACE} = $lh?$lh:"global"; + + Log3 ($hash, 3, "Log2Syslog $name - port $hash->{PORT}/$protocol opened for Syslog Collector on interface \"$hash->{INTERFACE}\""); + readingsSingleUpdate ($hash, "state", "initialized", 1); + delete($readyfnlist{"$name.$port"}); + $selectlist{"$name.$port"} = $hash; + +return; +} + +################################################################################################# +# Syslog Collector Daten empfangen +# (im Collector Model) +################################################################################################# +# called from the global loop, when the select for hash->{FD} reports data +sub Log2Syslog_Read($) { + my ($hash) = @_; + my $name = $hash->{NAME}; + my $socket = $hash->{SERVERSOCKET}; + my $st = ReadingsVal($name,"state","active"); + my $pp = AttrVal($name, "parseProfile", "IETF"); + my $mevt = AttrVal($name, "makeEvent", "intern"); # wie soll Reading/Eventerstellt werden + my $sevevt = AttrVal($name, "respectSeverity", ""); # welcher Schweregrad soll berücksichtigt werden (default: alle) + my ($err,$sev,$data,$ts,$phost,$pl,$ignore); + + return if(IsDisabled($name) || $hash->{MODEL} !~ /Collector/); + + if($pp =~ /BSD/) { + # BSD-Format + unless($socket->recv($data, $RFC3164len{DL})) { + # ungültige BSD-Payload + return if(length($data) == 0); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - received ".length($data)." bytes, but a BSD-message has to be 1024 bytes or less."); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); + $st = "receive error - see logfile"; + } else { + # parse Payload + ($err,$ignore,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); + $hash->{SEQNO}++; + if($err) { + $st = "parse error - see logfile"; + } elsif ($ignore) { + Log3 $name, 5, "Log2Syslog $name -> dataset was ignored by parseFn"; + } else { + return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen + $st = "active"; + if($mevt =~ /intern/) { + # kein Reading, nur Event + $pl = "$phost: $pl"; + Log2Syslog_Trigger($hash,$ts,$pl); + } elsif ($mevt =~ /reading/) { + # Reading, Event abhängig von event-on-.* + readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); + } else { + # Reading ohne Event + readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); + } + } + } + + } elsif($pp =~ /IETF/) { + # IETF-Format + unless($socket->recv($data, $RFC5425len{DL})) { + # ungültige IETF-Payload + return if(length($data) == 0); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - received ".length($data)." bytes, but a IETF-message has to be 8192 bytes or less."); + Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); + $st = "receive error - see logfile"; + } else { + # parse Payload + ($err,$ignore,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); + $hash->{SEQNO}++; + if($err) { + $st = "parse error - see logfile"; + } elsif ($ignore) { + Log3 $name, 5, "Log2Syslog $name -> dataset was ignored by parseFn"; + } else { + return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen + $st = "active"; + if($mevt =~ /intern/) { + # kein Reading, nur Event + $pl = "$phost: $pl"; + Log2Syslog_Trigger($hash,$ts,$pl); + } elsif ($mevt =~ /reading/) { + # Reading, Event abhängig von event-on-.* + readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); + } else { + # Reading ohne Event + readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); + } + } + } + } else { + # raw oder User eigenes Format + $socket->recv($data, 8192); + ($err,$ignore,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); + $hash->{SEQNO}++; + if($err) { + $st = "parse error - see logfile"; + } elsif ($ignore) { + Log3 $name, 5, "Log2Syslog $name -> dataset was ignored by parseFn"; + } else { + return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen + $st = "active"; + if($mevt =~ /intern/) { + # kein Reading, nur Event + $pl = "$phost: $pl"; + Log2Syslog_Trigger($hash,$ts,$pl); + } elsif ($mevt =~ /reading/) { + # Reading, Event abhängig von event-on-.* + readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); + } else { + # Reading ohne Event + readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); + } + } + + } + # readingsSingleUpdate($hash, "state", $st, 1) if($st ne OldValue($name)); + my $evt = ($st eq $hash->{HELPER}{OLDSTATE})?0:1; + readingsSingleUpdate($hash, "state", $st, $evt); + $hash->{HELPER}{OLDSTATE} = $st; + +return; +} + +############################################################################### +# Parsen Payload für Syslog-Server +# (im Collector Model) +############################################################################### +sub Log2Syslog_parsePayload($$) { + my ($hash,$data) = @_; + my $name = $hash->{NAME}; + my $pp = AttrVal($name, "parseProfile", "IETF"); + my $severity = ""; + my $facility = ""; + my @evf = split(",",AttrVal($name, "outputFields", "FAC,SEV,ID,CONT")); # auszugebene Felder im Event/Reading + my $ignore = 0; + my ($Mmm,$dd,$delimiter,$day,$ietf,$err,$pl,$tail); + + # Hash zur Umwandlung Felder in deren Variablen + my ($prival,$ts,$host,$date,$time,$id,$pid,$mid,$sdfield,$cont); + my $fac = ""; + my $sev = ""; + my %fh = (PRIVAL => \$prival, + FAC => \$fac, + SEV => \$sev, + TS => \$ts, + HOST => \$host, + DATE => \$date, + TIME => \$time, + ID => \$id, + PID => \$pid, + MID => \$mid, + SDFIELD => \$sdfield, + CONT => \$cont, + DATA => \$data + ); + + Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - ### new Syslog message Parsing ### "); + + # Sender Host / IP-Adresse ermitteln, $phost wird Reading im Event + my ($phost,$paddr) = Log2Syslog_evalPeer($hash); + $phost = $phost?$phost:$paddr; + + Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - raw message -> $data"); + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); # Istzeit Ableitung + $year = $year+1900; + + if ($pp =~ /raw/) { + Log2Syslog_Log3slog($name, 4, "$name - $data"); + $ts = TimeNow(); + $pl = $data; + + } elsif ($pp eq "BSD") { + # BSD Protokollformat https://tools.ietf.org/html/rfc3164 + # Beispiel data "<$prival>$month $day $time $myhost $id: $otp" + $data =~ /^<(?\d{1,3})>(?.*)$/; + $prival = $+{prival}; # must + $tail = $+{tail}; + $tail =~ /^((?\w{3})\s+(?\d{1,2})\s+(?