###################################################################################################################### # $Id: 93_Log2Syslog.pm 16688 2018-05-04 20:09:12Z DS_Starter $ ###################################################################################################################### # 93_Log2Syslog.pm # # (c) 2017-2018 by Heiko Maaz # e-mail: Heiko dot Maaz at t-online dot de # # This script is part of fhem. # # Fhem is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. # # Fhem is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with fhem. If not, see . # # The module is based on idea and input from betateilchen 92_rsyslog.pm # # Implements the Syslog Protocol of RFC 5424 https://tools.ietf.org/html/rfc5424 # and RFC 3164 https://tools.ietf.org/html/rfc3164 and # TLS Transport according to RFC5425 https://tools.ietf.org/pdf/rfc5425.pdf as well # ###################################################################################################################### # Versions History: # # 4.8.4 15.08.2018 BSD parsing changed # 4.8.3 14.08.2018 BSD setpayload changed, BSD parsing changed, Internal MYFQDN # 4.8.2 13.08.2018 rename makeMsgEvent to makeEvent # 4.8.1 12.08.2018 IETF-Syslog without VERSION changed, Log verbose 1 to 2 changed in parsePayload # 4.8.0 12.08.2018 enhanced IETF Parser to match logs without version # 4.7.0 10.08.2018 Parser for TPLink # 4.6.1 10.08.2018 some perl warnings, changed IETF Parser # 4.6.0 08.08.2018 set sendTestMessage added, Attribute "contDelimiter", "respectSeverity" # 4.5.1 07.08.2018 BSD Regex changed, setpayload of BSD changed # 4.5.0 06.08.2018 Regex capture groups used in parsePayload to set variables, parsing of BSD changed, # Attribute "makeMsgEvent" added # 4.4.0 04.08.2018 Attribute "outputFields" added # 4.3.0 03.08.2018 Attribute "parseFn" added # 4.2.0 03.08.2018 evaluate sender peer ip-address/hostname, use it as reading in event generation # 4.1.0 02.08.2018 state event generation changed # 4.0.0 30.07.2018 server mode (Collector) # 3.2.1 04.05.2018 fix compatibility with newer IO::Socket::SSL on debian 9, attr ssldebug for # debugging SSL messages # 3.2.0 22.11.2017 add NOTIFYDEV if possible # 3.1.0 28.08.2017 get-function added, commandref revised, $readingFnAttributes deleted # 3.0.0 27.08.2017 change attr type to protocol, ready to check in # 2.6.0 26.08.2017 more than one Log2Syslog device can be created # 2.5.2 26.08.2018 fix in splitting timestamp, change Log2Syslog_trate using internaltimer with attr # rateCalcRerun, function Log2Syslog_closesock # 2.5.1 24.08.2017 some fixes # 2.5.0 23.08.2017 TLS encryption available, new readings, $readingFnAttributes # 2.4.1 21.08.2017 changes in sub Log2Syslog_charfilter, change PROCID to $hash->{SEQNO} # switch to non-blocking in subs event/Log2Syslog_fhemlog # 2.4.0 20.08.2017 new sub Log2Syslog_Log3slog for entries in local fhemlog only -> verbose support # 2.3.1 19.08.2017 commandref revised # 2.3.0 18.08.2017 new parameter "ident" in DEF, sub setidex, Log2Syslog_charfilter # 2.2.0 17.08.2017 set BSD data length, set only acceptable characters (USASCII) in payload # commandref revised # 2.1.0 17.08.2017 sub Log2Syslog_opensock created # 2.0.0 16.08.2017 create syslog without SYS::SYSLOG # 1.1.1 13.08.2017 registrate Log2Syslog_fhemlog to %loginform in case of sending fhem-log # attribute timeout, commandref revised # 1.1.0 26.07.2017 add regex search to sub Log2Syslog_fhemlog # 1.0.0 25.07.2017 initial version package main; use strict; use warnings; use Scalar::Util qw(looks_like_number); use Encode qw(encode_utf8); use Net::Domain qw(hostname hostfqdn hostdomain); eval "use IO::Socket::INET;1" or my $MissModulSocket = "IO::Socket::INET"; eval "use Net::Domain qw(hostname hostfqdn hostdomain domainname);1" or my $MissModulNDom = "Net::Domain"; ############################################################################### # Forward declarations # sub Log2Syslog_Log3slog($$$); my $Log2SyslogVn = "4.8.4"; # Mappinghash BSD-Formatierung Monat my %Log2Syslog_BSDMonth = ( "01" => "Jan", "02" => "Feb", "03" => "Mar", "04" => "Apr", "05" => "May", "06" => "Jun", "07" => "Jul", "08" => "Aug", "09" => "Sep", "10" => "Oct", "11" => "Nov", "12" => "Dec", "Jan" => "01", "Feb" => "02", "Mar" => "03", "Apr" => "04", "May" => "05", "Jun" => "06", "Jul" => "07", "Aug" => "08", "Sep" => "09", "Oct" => "10", "Nov" => "11", "Dec" => "12" ); # Mappinghash Severity my %Log2Syslog_Severity = ( "0" => "Emergency", "1" => "Alert", "2" => "Critical", "3" => "Error", "4" => "Warning", "5" => "Notice", "6" => "Informational", "7" => "Debug", "Emergency" => "0", "Alert" => "1", "Critical" => "2", "Error" => "3", "Warning" => "4", "Notice" => "5", "Informational" => "6", "Debug" => "7" ); # Mappinghash Facility my %Log2Syslog_Facility = ( "0" => "kernel", "1" => "user", "2" => "mail", "3" => "system", "4" => "security", "5" => "syslog", "6" => "printer", "7" => "network", "8" => "UUCP", "9" => "clock", "10" => "security", "11" => "FTP", "12" => "NTP", "13" => "log_audit", "14" => "log_alert", "15" => "clock", "16" => "local0", "17" => "local1", "18" => "local2", "19" => "local3", "20" => "local4", "21" => "local5", "22" => "local6", "23" => "local7" ); # Längenvorgaben nach RFC3164 my %RFC3164len = ("TAG" => 32, # max. Länge TAG-Feld "DL" => 1024 # max. Lange Message insgesamt ); # Längenvorgaben nach RFC5425 my %RFC5425len = ("DL" => 8192, # max. Lange Message insgesamt mit TLS "HST" => 255, # max. Länge Hostname "ID" => 48, # max. Länge APP-NAME bzw. Ident "PID" => 128, # max. Länge Proc-ID "MID" => 32 # max. Länge MSGID ); ############################################################################### sub Log2Syslog_Initialize($) { my ($hash) = @_; $hash->{DefFn} = "Log2Syslog_Define"; $hash->{UndefFn} = "Log2Syslog_Undef"; $hash->{DeleteFn} = "Log2Syslog_Delete"; $hash->{SetFn} = "Log2Syslog_Set"; $hash->{GetFn} = "Log2Syslog_Get"; $hash->{AttrFn} = "Log2Syslog_Attr"; $hash->{NotifyFn} = "Log2Syslog_eventlog"; $hash->{ReadFn} = "Log2Syslog_Read"; $hash->{AttrList} = "addStateEvent:1,0 ". "disable:1,0,maintenance ". "addTimestamp:0,1 ". "contDelimiter ". "logFormat:BSD,IETF ". "makeEvent:no,intern,reading ". "outputFields:sortable-strict,PRIVAL,FAC,SEV,TS,HOST,DATE,TIME,ID,PID,MID,SDFIELD,CONT ". "parseProfile:BSD,IETF,TPLink-Switch,raw,ParseFn ". "parseFn:textField-long ". "respectSeverity:multiple-strict,Emergency,Alert,Critical,Error,Warning,Notice,Informational,Debug ". "ssldebug:0,1,2,3 ". "TLS:1,0 ". "timeout ". "protocol:UDP,TCP ". "port ". "rateCalcRerun ". $readingFnAttributes ; return undef; } ############################################################################### sub Log2Syslog_Define($@) { my ($hash, $def) = @_; my @a = split("[ \t][ \t]*", $def); my $name = $hash->{NAME}; return "Error: Perl module ".$MissModulSocket." is missing. Install it on Debian with: sudo apt-get install libio-socket-multicast-perl" if($MissModulSocket); return "Error: Perl module ".$MissModulNDom." is missing." if($MissModulNDom); # Example Sender: define splunklog Log2Syslog splunk.myds.me ident:Prod event:.* fhem:.* # Example Collector: define SyslogServer Log2Syslog delete($hash->{HELPER}{EVNTLOG}); delete($hash->{HELPER}{FHEMLOG}); delete($hash->{HELPER}{IDENT}); $hash->{MYFQDN} = hostfqdn(); # MYFQDN eigener Host (f. IETF) $hash->{MYHOST} = hostname(); # eigener Host (lt. RFC nur Hostname f. BSD) if(int(@a)-3 < 0){ # Einrichtung Servermode (Collector) Log3 ($name, 3, "Log2Syslog $name - entering Syslog servermode ..."); $hash->{MODEL} = "Collector"; Log2Syslog_initServer("$name,global"); } else { # Sendermode $hash->{MODEL} = "Sender"; Log2Syslog_setidrex($hash,$a[3]) if($a[3]); Log2Syslog_setidrex($hash,$a[4]) if($a[4]); Log2Syslog_setidrex($hash,$a[5]) if($a[5]); eval { "Hallo" =~ m/^$hash->{HELPER}{EVNTLOG}$/ } if($hash->{HELPER}{EVNTLOG}); return "Bad regexp: $@" if($@); eval { "Hallo" =~ m/^$hash->{HELPER}{FHEMLOG}$/ } if($hash->{HELPER}{FHEMLOG}); return "Bad regexp: $@" if($@); return "Bad regexp: starting with *" if((defined($hash->{HELPER}{EVNTLOG}) && $hash->{HELPER}{EVNTLOG} =~ m/^\*/) || (defined($hash->{HELPER}{FHEMLOG}) && $hash->{HELPER}{FHEMLOG} =~ m/^\*/)); # nur Events dieser Devices an NotifyFn weiterleiten, NOTIFYDEV wird gesetzt wenn möglich notifyRegexpChanged($hash, $hash->{HELPER}{EVNTLOG}) if($hash->{HELPER}{EVNTLOG}); $hash->{PEERHOST} = $a[2]; # Destination Host (Syslog Server) } $hash->{SEQNO} = 1; # PROCID in IETF, wird kontinuierlich hochgezählt $hash->{VERSION} = $Log2SyslogVn; $logInform{$hash->{NAME}} = "Log2Syslog_fhemlog"; # Funktion die in hash %loginform für $name eingetragen wird $hash->{HELPER}{SSLVER} = "n.a."; # Initialisierung $hash->{HELPER}{SSLALGO} = "n.a."; # Initialisierung $hash->{HELPER}{LTIME} = time(); # Init Timestmp f. Ratenbestimmung $hash->{HELPER}{OLDSEQNO} = $hash->{SEQNO}; # Init Sequenznummer f. Ratenbestimmung $hash->{HELPER}{OLDSTATE} = "initialized"; readingsBeginUpdate($hash); readingsBulkUpdate($hash, "SSL_Version", "n.a."); readingsBulkUpdate($hash, "SSL_Algorithm", "n.a."); readingsBulkUpdate($hash, "Transfered_logs_per_minute", 0); readingsBulkUpdate($hash, "state", "initialized") if($hash->{MODEL}=~/Sender/); readingsEndUpdate($hash,1); Log2Syslog_trate($hash); # regelm. Berechnung Transfer Rate starten return undef; } ################################################################################################# # Syslog Collector (Server-Mode) initialisieren # (im Collector Model) ################################################################################################# sub Log2Syslog_initServer($) { my ($a) = @_; my ($name,$global) = split(",",$a); my $hash = $defs{$name}; RemoveInternalTimer($hash, "Log2Syslog_initServer"); return if(IsDisabled($name)); if($init_done != 1) { InternalTimer(gettimeofday()+5, "Log2Syslog_initServer", "$name,$global", 0); return; } # Inititialisierung FHEM ist fertig -> Attribute geladen my $port = AttrVal($name, "TLS", 0)?AttrVal($name, "port", 6514):AttrVal($name, "port", 1514); my $protocol = lc(AttrVal($name, "protocol", "udp")); my $lh = $global ? ($global eq "global"? undef : $global) : ($hash->{IPV6} ? "::1" : "127.0.0.1"); Log3 $hash, 3, "Log2Syslog $name - Opening socket ..."; $hash->{SERVERSOCKET} = IO::Socket::INET->new( Domain => ($hash->{IPV6} ? AF_INET6() : AF_UNSPEC), # Linux bug LocalHost => $lh, Proto => $protocol, LocalPort => $port, ReuseAddr => 1 ); if(!$hash->{SERVERSOCKET}) { my $err = "Can't open Syslog Collector at $port: $!"; Log3 ($hash, 1, "Log2Syslog $name - $err"); readingsSingleUpdate ($hash, 'state', $err, 1); return; } $hash->{FD} = $hash->{SERVERSOCKET}->fileno(); $hash->{PORT} = $hash->{SERVERSOCKET}->sockport(); $hash->{PROTOCOL} = $protocol; $hash->{SEQNO} = 1; # PROCID wird kontinuierlich pro empfangenen Datensatz hochgezählt $hash->{HELPER}{OLDSEQNO} = $hash->{SEQNO}; # Init Sequenznummer f. Ratenbestimmung $hash->{INTERFACE} = $lh?$lh:"global"; Log3 ($hash, 3, "Log2Syslog $name - port $hash->{PORT}/$protocol opened for Syslog Collector on interface \"$hash->{INTERFACE}\""); readingsSingleUpdate ($hash, "state", "initialized", 1); delete($readyfnlist{"$name.$port"}); $selectlist{"$name.$port"} = $hash; return; } ################################################################################################# # Syslog Collector Daten empfangen # (im Collector Model) ################################################################################################# # called from the global loop, when the select for hash->{FD} reports data sub Log2Syslog_Read($) { my ($hash) = @_; my $name = $hash->{NAME}; my $socket = $hash->{SERVERSOCKET}; my $st = ReadingsVal($name,"state","active"); my $pp = AttrVal($name, "parseProfile", "IETF"); my $mevt = AttrVal($name, "makeEvent", "intern"); # wie soll Reading/Eventerstellt werden my $sevevt = AttrVal($name, "respectSeverity", ""); # welcher Schweregrad soll berücksichtigt werden (default: alle) my ($err,$sev,$data,$ts,$phost,$pl); return if(IsDisabled($name) || $hash->{MODEL} !~ /Collector/); if($pp =~ /BSD/) { # BSD-Format unless($socket->recv($data, $RFC3164len{DL})) { # ungültige BSD-Payload return if(length($data) == 0); Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - received ".length($data)." bytes, but a BSD-message has to be 1024 bytes or less."); Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); $st = "receive error - see logfile"; } else { # parse Payload ($err,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); $hash->{SEQNO}++; if($err) { $st = "parse error - see logfile"; } else { return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen $st = "active"; if($mevt =~ /intern/) { # kein Reading, nur Event $pl = "$phost: $pl"; Log2Syslog_Trigger($hash,$ts,$pl); } elsif ($mevt =~ /reading/) { # Reading, Event abhängig von event-on-.* readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); } else { # Reading ohne Event readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); } } } } elsif($pp =~ /IETF/) { # IETF-Format unless($socket->recv($data, $RFC5425len{DL})) { # ungültige IETF-Payload return if(length($data) == 0); Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - received ".length($data)." bytes, but a IETF-message has to be 8192 bytes or less."); Log2Syslog_Log3slog ($hash, 3, "Log2Syslog $name - Seq \"$hash->{SEQNO}\" invalid data: $data"); $st = "receive error - see logfile"; } else { # parse Payload ($err,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); $hash->{SEQNO}++; if($err) { $st = "parse error - see logfile"; } else { return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen $st = "active"; if($mevt =~ /intern/) { # kein Reading, nur Event $pl = "$phost: $pl"; Log2Syslog_Trigger($hash,$ts,$pl); } elsif ($mevt =~ /reading/) { # Reading, Event abhängig von event-on-.* readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); } else { # Reading ohne Event readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); } } } } else { # raw oder User eigenes Format $socket->recv($data, 8192); ($err,$sev,$phost,$ts,$pl) = Log2Syslog_parsePayload($hash,$data); $hash->{SEQNO}++; if($err) { $st = "parse error - see logfile"; } else { return if($sevevt && $sevevt !~ m/$sev/); # Message nicht berücksichtigen $st = "active"; if($mevt =~ /intern/) { # kein Reading, nur Event $pl = "$phost: $pl"; Log2Syslog_Trigger($hash,$ts,$pl); } elsif ($mevt =~ /reading/) { # Reading, Event abhängig von event-on-.* readingsSingleUpdate($hash, "MSG_$phost", $pl, 1); } else { # Reading ohne Event readingsSingleUpdate($hash, "MSG_$phost", $pl, 0); } } } # readingsSingleUpdate($hash, "state", $st, 1) if($st ne OldValue($name)); my $evt = ($st eq $hash->{HELPER}{OLDSTATE})?0:1; readingsSingleUpdate($hash, "state", $st, $evt); $hash->{HELPER}{OLDSTATE} = $st; return; } ############################################################################### # Parsen Payload für Syslog-Server # (im Collector Model) ############################################################################### sub Log2Syslog_parsePayload($$) { my ($hash,$data) = @_; my $name = $hash->{NAME}; my $pp = AttrVal($name, "parseProfile", "IETF"); my $severity = ""; my $facility = ""; my @evf = split(",",AttrVal($name, "outputFields", "FAC,SEV,ID,CONT")); # auszugebene Felder im Event/Reading my ($Mmm,$dd,$delimiter,$day,$ietf,$err,$pl,$tail); # Hash zur Umwandlung Felder in deren Variablen my ($prival,$ts,$host,$date,$time,$id,$pid,$mid,$sdfield,$cont); my $fac = ""; my $sev = ""; my %fh = (PRIVAL => \$prival, FAC => \$fac, SEV => \$sev, TS => \$ts, HOST => \$host, DATE => \$date, TIME => \$time, ID => \$id, PID => \$pid, MID => \$mid, SDFIELD => \$sdfield, CONT => \$cont, DATA => \$data ); Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - ### new Syslog message Parsing ### "); # Sender Host / IP-Adresse ermitteln, $phost wird Reading im Event my ($phost,$paddr) = Log2Syslog_evalPeer($hash); $phost = $phost?$phost:$paddr; Log2Syslog_Log3slog ($hash, 5, "Log2Syslog $name - raw message -> $data"); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); # Istzeit Ableitung $year = $year+1900; if ($pp =~ /raw/) { Log2Syslog_Log3slog($name, 4, "$name - $data"); $ts = TimeNow(); $pl = $data; } elsif ($pp eq "BSD") { # BSD Protokollformat https://tools.ietf.org/html/rfc3164 # Beispiel data "<$prival>$month $day $time $myhost $id: $otp" $data =~ /^<(?\d{1,3})>(?.*)$/; $prival = $+{prival}; # must $tail = $+{tail}; $tail =~ /^((?\w{3})\s+(?\d{1,2})\s+(?