diff --git a/fhem/CHANGED b/fhem/CHANGED index d84af34eb..f36e6a7bf 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,7 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - bugfix: 93_DbLog.pm: Version is now 2.9.1, "MySQL Server has gone", + new commands purgeCache, commitCache - bugfix: 74_XiaomiFlowerSens: fix disableForInterval Bug - bugfix: 88_HMCCU.pm: fixed attribute substexcl - update: 88_HMCCU.pm: new version 3.7 diff --git a/fhem/FHEM/93_DbLog.pm b/fhem/FHEM/93_DbLog.pm index 356d900c7..9b9bea7da 100644 --- a/fhem/FHEM/93_DbLog.pm +++ b/fhem/FHEM/93_DbLog.pm @@ -13,6 +13,11 @@ ############################################################################################################################ # Versions History done by DS_Starter: # +# 2.9.1 14.01.2017 changed DbLog_ParseEvent to CallInstanceFn, renamed flushCache to purgeCache, +# renamed syncCache to commitCache, attr cacheEvents changed to 0,1,2 +# 2.9 11.01.2017 changed DbLog_ParseEvent to CallFn +# 2.8.9 11.01.2017 own $dbhp (new DbLog_ConnectPush) for synchronous logging, delete $hash->{HELPER}{RUNNING_PID} +# if DEAD, add func flushCache, syncCache # 2.8.8 10.01.2017 connection check in Get added, avoid warning "commit/rollback ineffective with AutoCommit enabled" # 2.8.7 10.01.2017 bugfix no dropdown list in SVG if asynchronous mode activated (func DbLog_sampleDataFn) # 2.8.6 09.01.2017 Workaround for Warning begin_work failed: Turning off AutoCommit failed, start new timer of @@ -55,7 +60,7 @@ use Data::Dumper; use Blocking; use Time::HiRes qw(gettimeofday tv_interval); -my $DbLogVersion = "2.8.8"; +my $DbLogVersion = "2.9.1"; my %columns = ("DEVICE" => 64, "TYPE" => 64, @@ -80,7 +85,7 @@ sub DbLog_Initialize($) $hash->{AttrFn} = "DbLog_Attr"; $hash->{SVG_regexpFn} = "DbLog_regexpFn"; $hash->{ShutdownFn} = "DbLog_Shutdown"; - $hash->{AttrList} = "disable:0,1 ". + $hash->{AttrList} = "disable:1,0 ". "DbLogType:Current,History,Current/History ". "shutdownWait ". "suppressUndef:0,1 ". @@ -90,7 +95,7 @@ sub DbLog_Initialize($) "noNotifyDev:1,0 ". "showproctime:1,0 ". "asyncMode:1,0 ". - "cacheEvents:1,0 ". + "cacheEvents:2,1,0 ". "syncEvents:1,0 ". "DbLogSelectionMode:Exclude,Include,Exclude/Include ". $readingFnAttributes; @@ -149,6 +154,7 @@ sub DbLog_Define($@) readingsSingleUpdate($hash, 'state', 'waiting for connection', 1); eval { DbLog_Connect($hash); }; + eval { DbLog_ConnectPush($hash); }; DbLog_execmemcache($hash); @@ -207,6 +213,7 @@ sub DbLog_Attr(@) { } else { $hash->{MODE} = "synchronous"; delete($defs{$name}{READINGS}{NextSync}); + delete($defs{$name}{READINGS}{CacheUsage}); delete($defs{$name}{READINGS}{background_processing_time}); delete($defs{$name}{READINGS}{sql_processing_time}); DbLog_execmemcache($hash); @@ -253,7 +260,8 @@ return undef; sub DbLog_Set($@) { my ($hash, @a) = @_; my $name = $hash->{NAME}; - my $usage = "Unknown argument, choose one of reduceLog reopen:noArg rereadcfg:noArg count:noArg deleteOldDays userCommand listCache:noArg"; + my $usage = "Unknown argument, choose one of reduceLog reopen:noArg rereadcfg:noArg count:noArg deleteOldDays userCommand + listCache:noArg purgeCache:noArg commitCache:noArg"; return $usage if(int(@a) < 2); my $dbh = $hash->{DBH}; my $ret; @@ -297,6 +305,13 @@ sub DbLog_Set($@) { DbLog_Connect($hash); $ret = "Rereadcfg executed."; } + elsif ($a[1] eq 'purgeCache') { + delete $hash->{cache}; + readingsSingleUpdate($hash, 'CacheUsage', 0, 1); + } + elsif ($a[1] eq 'commitCache') { + DbLog_execmemcache($hash); + } elsif ($a[1] eq 'listCache') { my $cache; foreach my $key (sort(keys%{$hash->{cache}{memcache}})) { @@ -400,18 +415,11 @@ sub DbLog_ParseEvent($$$) my $value; my $unit; - my $dtype = $defs{$device}{TYPE}; - - if($modules{$dtype}{DbLog_splitFn}) { - # let the module do the job! - - Log3 $device, 5, "DbLog_ParseEvent calling external DbLog_splitFn for type: $dtype , device: $device"; - no strict "refs"; - ($reading,$value,$unit) = - &{$modules{$dtype}{DbLog_splitFn}}($event, $device); - use strict "refs"; - @result= ($reading,$value,$unit); - return @result; + # Splitfunktion der Eventquelle aufrufen + ($reading, $value, $unit) = CallInstanceFn($device, "DbLog_splitFn", $event, $device); + # undef bedeutet, Modul stellt keine DbLog_splitFn bereit + if($reading) { + return ($reading, $value, $unit); } # split the event into reading, value and unit @@ -827,7 +835,7 @@ sub DbLog_Log($$) { } my $row = ($timestamp."|".$dev_name."|".$dev_type."|".$event."|".$reading."|".$value."|".$unit); - Log3 $hash->{NAME}, 4, "DbLog $name -> added event to memcache - Timestamp: $timestamp, Device: $dev_name, Type: $dev_type, Event: $event, Reading: $reading, Value: $value, Unit: $unit" + Log3 $hash->{NAME}, 4, "DbLog $name -> added event - Timestamp: $timestamp, Device: $dev_name, Type: $dev_type, Event: $event, Reading: $reading, Value: $value, Unit: $unit" if($vb4show); if($async) { @@ -836,6 +844,14 @@ sub DbLog_Log($$) { $hash->{cache}{index}++; my $index = $hash->{cache}{index}; $hash->{cache}{memcache}{$index} = $row; + + my $memcount = $hash->{cache}{memcache}?scalar(keys%{$hash->{cache}{memcache}}):0; + if(AttrVal($name, "cacheEvents", undef) && AttrVal($name, "cacheEvents", undef) == 1) { + readingsSingleUpdate($hash, "CacheUsage", $memcount, 1) + } else { + readingsSingleUpdate($hash, "CacheUsage", $memcount, 0) + } + } else { # synchoner Mode push(@row_array, $row); @@ -844,15 +860,7 @@ sub DbLog_Log($$) { } } }; - if($async) { - # asynchoner non-blocking Mode - my $memcount = $hash->{cache}{memcache}?scalar(keys%{$hash->{cache}{memcache}}):0; - if(AttrVal($name, "cacheEvents", undef)) { - readingsSingleUpdate($hash, "CacheUsage", $memcount, 1) - } else { - readingsSingleUpdate($hash, "CacheUsage", $memcount, 0) - } - } else { + if(!$async) { if(@row_array) { # synchoner Mode my $error = DbLog_Push($hash, $vb4show, @row_array); @@ -874,26 +882,28 @@ return; ################################################################################################# sub DbLog_Push(@) { my ($hash, $vb4show, @row_array) = @_; - my $dbh = $hash->{DBH}; + my $dbhp = $hash->{DBHP}; my $name = $hash->{NAME}; my $DbLogType = AttrVal($name, "DbLogType", "History"); my $error = 0; my $doins = 0; # Hilfsvariable, wenn "1" sollen inserts in Tabele current erfolgen (updates schlugen fehl) eval { - if ( !$dbh || not $dbh->ping ) { + if ( !$dbhp || not $dbhp->ping ) { #### DB Session dead, try to reopen now ! - DbLog_Connect($hash); + DbLog_ConnectPush($hash); } }; if ($@) { - Log3($name, 1, "DbLog $name: DBLog_Push - DB Session dead! - $@"); - return $@; + Log3($name, 1, "DbLog $name: DBLog_Push - DB Session dead! - $@"); + return $@; + } else { + $dbhp = $hash->{DBHP}; } - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; + $dbhp->{RaiseError} = 1; + $dbhp->{PrintError} = 0; my (@timestamp,@device,@type,@event,@reading,@value,@unit); my (@timestamp_cur,@device_cur,@type_cur,@event_cur,@reading_cur,@value_cur,@unit_cur); @@ -918,7 +928,7 @@ sub DbLog_Push(@) { if (lc($DbLogType) =~ m(history)) { # for insert history - $sth_ih = $dbh->prepare("INSERT INTO history (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); + $sth_ih = $dbhp->prepare("INSERT INTO history (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); $sth_ih->bind_param_array(1, [@timestamp]); $sth_ih->bind_param_array(2, [@device]); $sth_ih->bind_param_array(3, [@type]); @@ -930,10 +940,10 @@ sub DbLog_Push(@) { if (lc($DbLogType) =~ m(current) ) { # for insert current - $sth_ic = $dbh->prepare("INSERT INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); + $sth_ic = $dbhp->prepare("INSERT INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); # for update current - $sth_uc = $dbh->prepare("UPDATE current SET TIMESTAMP=?, TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (DEVICE=?) AND (READING=?)"); + $sth_uc = $dbhp->prepare("UPDATE current SET TIMESTAMP=?, TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (DEVICE=?) AND (READING=?)"); $sth_uc->bind_param_array(1, [@timestamp]); $sth_uc->bind_param_array(2, [@type]); $sth_uc->bind_param_array(3, [@event]); @@ -943,7 +953,7 @@ sub DbLog_Push(@) { $sth_uc->bind_param_array(7, [@reading]); } - eval {$dbh->begin_work();}; # issue: begin_work failed: Turning off AutoCommit failed + eval {$dbhp->begin_work();}; # issue: begin_work failed: Turning off AutoCommit failed my ($tuples, $rows); eval { # insert into history @@ -957,7 +967,7 @@ sub DbLog_Push(@) { my $status = $tuple_status[$tuple]; $status = 0 if($status eq "0E0"); next if($status); # $status ist "1" wenn insert ok - Log3 $hash->{NAME}, 2, "DbLog $name -> Failed to insert into history: $device[$tuple], $event[$tuple]" if($vb4show); + Log3 $hash->{NAME}, 2, "DbLog $name -> Failed to insert into history: $device[$tuple], $event[$tuple]"; } } } @@ -1016,15 +1026,13 @@ sub DbLog_Push(@) { if ($@) { Log3 $hash->{NAME}, 2, "DbLog $name -> Error: $@"; - $dbh->rollback() if(!$dbh->{AutoCommit}); + $dbhp->rollback() if(!$dbhp->{AutoCommit}); $error = $@; - $dbh->disconnect(); - DbLog_Connect($hash); } else { - $dbh->commit() if(!$dbh->{AutoCommit}); - $dbh->{RaiseError} = 0; - $dbh->{PrintError} = 1; + $dbhp->commit() if(!$dbhp->{AutoCommit}); + $dbhp->{RaiseError} = 0; + $dbhp->{PrintError} = 1; } return $error; @@ -1057,8 +1065,13 @@ sub DbLog_execmemcache ($) { if(!$async || IsDisabled($name)) { return; } - - # nur Verbindungstest, DbLog_PushAsyncDone hat eigene Verbindungsroutine + + # tote PID löschen + if($hash->{HELPER}{RUNNING_PID} && $hash->{HELPER}{RUNNING_PID}{pid} =~ m/DEAD/) { + delete $hash->{HELPER}{RUNNING_PID}; + } + + # nur Verbindungstest, DbLog_PushAsync hat eigene Verbindungsroutine eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1 }); }; @@ -1070,6 +1083,12 @@ sub DbLog_execmemcache ($) { # Testverbindung abbauen $dbh->disconnect(); $memcount = $hash->{cache}{memcache}?scalar(keys%{$hash->{cache}{memcache}}):0; + + if(AttrVal($name, "cacheEvents", undef) && AttrVal($name, "cacheEvents", undef) == 2) { + readingsSingleUpdate($hash, "CacheUsage", $memcount, 1) + } else { + readingsSingleUpdate($hash, "CacheUsage", $memcount, 0) + } if($memcount && !$hash->{HELPER}{RUNNING_PID}) { Log3 $name, 5, "DbLog $name -> ################################################################"; @@ -1101,16 +1120,10 @@ sub DbLog_execmemcache ($) { } } - $memcount = scalar(keys%{$hash->{cache}{memcache}}); + # $memcount = scalar(keys%{$hash->{cache}{memcache}}); my $nextsync = gettimeofday()+$syncival; my $nsdt = FmtDateTime($nextsync); - - if(AttrVal($name, "cacheEvents", undef)) { - readingsSingleUpdate($hash, "CacheUsage", $memcount, 1) - } else { - readingsSingleUpdate($hash, "CacheUsage", $memcount, 0) - } if(AttrVal($name, "syncEvents", undef)) { readingsSingleUpdate($hash, "NextSync", $nsdt, 1); @@ -1343,10 +1356,15 @@ sub DbLog_PushAsyncDone ($) { Log3 ($name, 5, "DbLog $name -> Start DbLog_PushAsyncDone"); $state = "disabled" if(IsDisabled($name)); - readingsBeginUpdate($hash); - readingsBulkUpdate($hash, "background_processing_time", sprintf("%.4f",$brt)) if(AttrVal($name, "showproctime", undef)); - readingsBulkUpdate($hash, "sql_processing_time", sprintf("%.4f",$rt)) if(AttrVal($name, "showproctime", undef)); - readingsEndUpdate($hash, 1); + $memcount = $hash->{cache}{memcache}?scalar(keys%{$hash->{cache}{memcache}}):0; + readingsSingleUpdate($hash, 'CacheUsage', $memcount, 0); + + if(AttrVal($name, "showproctime", undef)) { + readingsBeginUpdate($hash); + readingsBulkUpdate($hash, "background_processing_time", sprintf("%.4f",$brt)); + readingsBulkUpdate($hash, "sql_processing_time", sprintf("%.4f",$rt)); + readingsEndUpdate($hash, 1); + } if($error) { readingsSingleUpdate($hash, "state", $error, 1); @@ -1481,15 +1499,16 @@ sub DbLog_Connect($) $hash->{DBH}= $dbh; - if ($hash->{DBMODEL} eq "SQLITE") { - $dbh->do("PRAGMA temp_store=MEMORY"); - $dbh->do("PRAGMA synchronous=NORMAL"); - $dbh->do("PRAGMA journal_mode=WAL"); - $dbh->do("PRAGMA cache_size=4000"); - $dbh->do("CREATE TEMP TABLE IF NOT EXISTS current (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); - $dbh->do("CREATE TABLE IF NOT EXISTS history (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); - $dbh->do("CREATE INDEX IF NOT EXISTS Search_Idx ON `history` (DEVICE, READING, TIMESTAMP)"); - } +# nach sub DbLog_ConnectPush verschoben +# if ($hash->{DBMODEL} eq "SQLITE") { +# $dbh->do("PRAGMA temp_store=MEMORY"); +# $dbh->do("PRAGMA synchronous=NORMAL"); +# $dbh->do("PRAGMA journal_mode=WAL"); +# $dbh->do("PRAGMA cache_size=4000"); +# $dbh->do("CREATE TEMP TABLE IF NOT EXISTS current (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); +# $dbh->do("CREATE TABLE IF NOT EXISTS history (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); +# $dbh->do("CREATE INDEX IF NOT EXISTS Search_Idx ON `history` (DEVICE, READING, TIMESTAMP)"); +# } # no webfrontend connection for plotfork return 1 if( $hash->{PID} != $$ ); @@ -1516,6 +1535,44 @@ sub DbLog_Connect($) return 1; } +sub DbLog_ConnectPush($) { + # own $dbhp for synchronous logging + my ($hash)= @_; + my $name = $hash->{NAME}; + my $dbconn = $hash->{dbconn}; + my $dbuser = $hash->{dbuser}; + my $dbpassword = $attr{"sec$name"}{secret}; + + Log3 $hash->{NAME}, 3, "Creating Push-Handle to database $dbconn with user $dbuser"; + my $dbhp = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0 }); + + if(!$dbhp) { + RemoveInternalTimer($hash, "DbLog_ConnectPush"); + Log3 $hash->{NAME}, 4, 'DbLog: Trying to connect to database'; + readingsSingleUpdate($hash, 'state', 'disconnected', 1); + InternalTimer(time+5, 'DbLog_ConnectPush', $hash, 0); + Log3 $hash->{NAME}, 4, 'Waiting for database connection'; + return 0; + } + + Log3 $hash->{NAME}, 3, "Push-Handle to db $dbconn created"; + readingsSingleUpdate($hash, 'state', 'connected', 1); + + $hash->{DBHP}= $dbhp; + + if ($hash->{DBMODEL} eq "SQLITE") { + $dbhp->do("PRAGMA temp_store=MEMORY"); + $dbhp->do("PRAGMA synchronous=NORMAL"); + $dbhp->do("PRAGMA journal_mode=WAL"); + $dbhp->do("PRAGMA cache_size=4000"); + $dbhp->do("CREATE TEMP TABLE IF NOT EXISTS current (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); + $dbhp->do("CREATE TABLE IF NOT EXISTS history (TIMESTAMP TIMESTAMP, DEVICE varchar(64), TYPE varchar(64), EVENT varchar(512), READING varchar(64), VALUE varchar(128), UNIT varchar(32))"); + $dbhp->do("CREATE INDEX IF NOT EXISTS Search_Idx ON `history` (DEVICE, READING, TIMESTAMP)"); + } + + return 1; +} + ################################################################ # # Prozeduren zum Ausfuehren des SQLs @@ -2723,6 +2780,10 @@ sub dbReadings($@) { to avoid storing the password in the main configuration file and to have it visible in the output of the list command.

+ + DbLog distinguishes between the synchronous (default) and asynchronous logmode. The logmode is adjustable by the + attribute asyncMode. +

The modules DBI and DBD::<dbtype> need to be installed (use cpan -i <module> @@ -2774,6 +2835,12 @@ sub dbReadings($@) { Set
@@ -3216,13 +3291,17 @@ sub dbReadings($@) { definiert in <configfilename>. (Vergleiche Beipspielkonfigurationsdatei in contrib/dblog/db.conf).
Die Konfiguration ist in einer sparaten Datei abgelegt um das Datenbankpasswort - nicht in Klartext in der FHEM-Haupt-Konfigurationsdatei speichern zu mössen. - Ansonsten wäre es mittels des list + nicht in Klartext in der FHEM-Haupt-Konfigurationsdatei speichern zu müssen. + Ansonsten wäre es mittels des list Befehls einfach auslesbar.

+ + DbLog unterscheidet den synchronen (Default) und asynchronen Logmodus. Der Logmodus ist über das + Attribut asyncMode einstellbar. +

- Die Perl-Module DBI and DBD::<dbtype> - mössen installiert werden (use cpan -i <module> + Die Perl-Module DBI und DBD::<dbtype> + müssen installiert werden (use cpan -i <module> falls die eigene Distribution diese nicht schon mitbringt).

@@ -3271,6 +3350,12 @@ sub dbReadings($@) { Set