diff --git a/fhem/CHANGED b/fhem/CHANGED index 4c9597ac1..f1ac989ec 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,8 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_DbRep: add column header for custom selects (sqlCmd), + new sqlSpecial readingsDifferenceByTimeDelta for MySQL, + bugfix: FHEM crash if no time-attribute is set and time - change: 70_BOTVAC: use keepalive for status update refactor connection handling fix warnings while loading maps diff --git a/fhem/FHEM/93_DbRep.pm b/fhem/FHEM/93_DbRep.pm index 561f54a06..46939d98b 100644 --- a/fhem/FHEM/93_DbRep.pm +++ b/fhem/FHEM/93_DbRep.pm @@ -58,6 +58,9 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # Version History intern our %DbRep_vNotesIntern = ( + "8.38.0" => "21.03.2020 sqlSpecial readingsDifferenceByTimeDelta, fix FHEM crash if no time-attribute is set and time ". + "option of delEntries / reduceLog is used, minor fix for sqlCmdHistory ", + "8.37.0" => "20.03.2020 add column header for free selects (sqlCmd) ", "8.36.0" => "19.03.2020 sqlSpecial recentReadingsOfDevice ", "8.35.0" => "19.03.2020 option older than days and newer than days for delEntries function ", "8.34.0" => "18.03.2020 option writeToDBSingle for functions averageValue, sumValue ", @@ -191,6 +194,8 @@ our %DbRep_vNotesIntern = ( # Version History extern: our %DbRep_vNotesExtern = ( + "8.36.0" => "19.03.2020 Some simplifications for fast mutable module usage is built in, such as possible command option older than / newer than days for reduceLog and delEntries function. ". + "A new option \"writeToDBSingle\" for functions averageValue, sumValue is built in as well as a new sqlSpecial \"recentReadingsOfDevice\". ", "8.32.0" => "29.01.2020 A new option \"deleteOther\" is available for commands \"maxValue\" and \"minValue\". Now it is possible to delete all values except the ". "extreme values (max/min) from the database within the specified limits of time, device, reading and so on. ", "8.30.4" => "22.01.2020 The behavior of write back values to database is changed for functions averageValue and sumValue. The solution values of that functions now are ". @@ -570,7 +575,7 @@ sub DbRep_Set($@) { (($hash->{ROLE} ne "Agent")?"reduceLog ":""). (($hash->{ROLE} ne "Agent")?"sqlCmd:textField-long ":""). (($hash->{ROLE} ne "Agent" && $hl)?"sqlCmdHistory:".$hl." ":""). - (($hash->{ROLE} ne "Agent")?"sqlSpecial:50mostFreqLogsLast2days,allDevCount,allDevReadCount,recentReadingsOfDevice ":""). + (($hash->{ROLE} ne "Agent")?"sqlSpecial:50mostFreqLogsLast2days,allDevCount,allDevReadCount,recentReadingsOfDevice".(($dbmodel eq "MYSQL")?",readingsDifferenceByTimeDelta":"")." ":""). (($hash->{ROLE} ne "Agent")?"syncStandby ":""). (($hash->{ROLE} ne "Agent")?"tableCurrentFillup:noArg ":""). (($hash->{ROLE} ne "Agent")?"tableCurrentPurge:noArg ":""). @@ -907,10 +912,13 @@ sub DbRep_Set($@) { # $sqlcmd =~ tr/ A-Za-z0-9!"#$§%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; # V8.36.0 20.03.2020 } if($opt eq "sqlCmdHistory") { - $prop =~ tr/ A-Za-z0-9!"#$%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; + $prop =~ s/§/_ESC_ECS_/g; + $prop =~ tr/ A-Za-z0-9!"#%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; + $prop =~ s/_ESC_ECS_/§/g; $prop =~ s//,/g; # noch aus Kompatibilitätsgründen enthalten $prop =~ s/(\x20)*\xbc/,/g; # Forum: https://forum.fhem.de/index.php/topic,103908.0.html $sqlcmd = $prop; + Log3 ($name, 1, "DbRep $name - com: $sqlcmd"); if($sqlcmd eq "___purge_historylist___") { delete($hash->{HELPER}{SQLHIST}); DbRep_setCmdFile($name."_sqlCmdList","",$hash); # Löschen der sql History Liste im DbRep-Keyfile @@ -1954,15 +1962,16 @@ sub DbRep_Main($$;$) { # zentrales Timestamp-Array und Zeitgrenzen bereitstellen my ($epoch_seconds_begin,$epoch_seconds_end,$runtime_string_first,$runtime_string_next); my $ts = "no_aggregation"; # Dummy für eine Select-Schleife wenn != $IsTimeSet || $IsAggrSet - my ($IsTimeSet,$IsAggrSet,$aggregation) = DbRep_checktimeaggr($hash); + my ($IsTimeSet,$IsAggrSet,$aggregation) = DbRep_checktimeaggr($hash); + if($IsTimeSet || $IsAggrSet) { ($epoch_seconds_begin,$epoch_seconds_end,$runtime_string_first,$runtime_string_next,$ts) = DbRep_createTimeArray($hash,$aggregation,$opt); } else { Log3 ($name, 4, "DbRep $name - Timestamp begin human readable: not set") if($opt !~ /tableCurrentPurge/); Log3 ($name, 4, "DbRep $name - Timestamp end human readable: not set") if($opt !~ /tableCurrentPurge/); } - - Log3 ($name, 4, "DbRep $name - Aggregation: $aggregation") if($opt !~ /tableCurrentPurge|tableCurrentFillup|fetchrows|insert|reduceLog/); + + Log3 ($name, 4, "DbRep $name - Aggregation: $aggregation") if($opt !~ /tableCurrentPurge|tableCurrentFillup|fetchrows|insert|reduceLog|delEntries/); ##### Funktionsaufrufe ##### if ($opt eq "sumValue") { @@ -2046,16 +2055,20 @@ sub DbRep_Main($$;$) { } elsif ($opt =~ /sqlCmd|sqlSpecial/ ) { # Execute a generic sql command or special sql if ($opt =~ /sqlSpecial/) { + my ($tq,$gcl); + if($prop eq "50mostFreqLogsLast2days") { $prop = "select Device, reading, count(0) AS `countA` from history where ( TIMESTAMP > (now() - interval 2 day)) group by DEVICE, READING order by countA desc, DEVICE limit 50;" if($dbmodel =~ /MYSQL/); $prop = "select Device, reading, count(0) AS `countA` from history where ( TIMESTAMP > ('now' - '2 days')) group by DEVICE, READING order by countA desc, DEVICE limit 50;" if($dbmodel =~ /SQLITE/); $prop = "select Device, reading, count(0) AS countA from history where ( TIMESTAMP > (NOW() - INTERVAL '2' DAY)) group by DEVICE, READING order by countA desc, DEVICE limit 50;" if($dbmodel =~ /POSTGRESQL/); + } elsif ($prop eq "allDevReadCount") { $prop = "select device, reading, count(*) from history group by DEVICE, READING;"; + } elsif ($prop eq "allDevCount") { $prop = "select device, count(*) from history group by DEVICE;"; + } elsif ($prop eq "recentReadingsOfDevice") { - my ($tq,$gcl); if($dbmodel =~ /MYSQL/) {$tq = "NOW() - INTERVAL 1 DAY"; $gcl = "READING"}; if($dbmodel =~ /SQLITE/) {$tq = "datetime('now','-1 day')"; $gcl = "READING"}; if($dbmodel =~ /POSTGRESQL/) {$tq = "CURRENT_TIMESTAMP - INTERVAL '1 day'"; $gcl = "READING,DEVICE"}; @@ -2070,6 +2083,26 @@ sub DbRep_Main($$;$) { x.READING = t1.READING;"); $prop = join(" ", @cmd); + + } elsif ($prop eq "readingsDifferenceByTimeDelta") { + my @cmd = split(/\s/, "SET \@diff=0; + SET \@delta=NULL; + SELECT t1.TIMESTAMP,t1.READING,t1.VALUE,t1.DIFF,t1.TIME_DELTA + FROM (SELECT TIMESTAMP,READING,VALUE, + cast((VALUE-\@diff) AS DECIMAL(12,4)) AS DIFF, + \@diff:=VALUE AS curr_V, + TIMESTAMPDIFF(MINUTE,\@delta,TIMESTAMP) AS TIME_DELTA, + \@delta:=TIMESTAMP AS curr_T + FROM history + WHERE DEVICE = '".$device."' AND + READING = '".$reading."' AND + TIMESTAMP >= §timestamp_begin§ AND + TIMESTAMP <= §timestamp_end§ + ORDER BY TIMESTAMP + ) t1;" + ); + + $prop = join(" ", @cmd); } } $hash->{HELPER}{RUNNING_PID} = BlockingCall("sqlCmd_DoParse", "$name|$opt|$runtime_string_first|$runtime_string_next|$prop", "sqlCmd_ParseDone", $to, "DbRep_ParseAborted", $hash); @@ -6179,33 +6212,39 @@ sub sqlCmd_DoParse($) { return "$name|''|$opt|$sql|''|''|$err"; } - my @rows; + my (@rows,$row,@head); my $nrows = 0; if($sql =~ m/^\s*(explain|select|pragma|show)/is) { - while (my @line = $sth->fetchrow_array()) { - Log3 ($name, 4, "DbRep $name - SQL result: @line"); - my $row = join("$srs", @line); + @head = map { uc($sth->{NAME}[$_]) } keys @{$sth->{NAME}}; # https://metacpan.org/pod/DBI#NAME1 + if (@head) { + $row = join("$srs", @head); + push(@rows, $row); + } - # join Delimiter "§" escapen - $row =~ s/§/|°escaped°|/g; + while (my @line = $sth->fetchrow_array()) { + Log3 ($name, 4, "DbRep $name - SQL result: @line"); + $row = join("$srs", @line); - push(@rows, $row); - # Anzahl der Datensätze - $nrows++; - } + # join Delimiter "§" escapen + $row =~ s/§/|°escaped°|/g; + + push(@rows, $row); + # Anzahl der Datensätze + $nrows++; + } } else { - $nrows = $sth->rows; - eval {$dbh->commit() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@,""); - Log3 ($name, 2, "DbRep $name - ERROR - $@"); - $dbh->disconnect; - return "$name|''|$opt|$sql|''|''|$err"; - } + $nrows = $sth->rows; + eval {$dbh->commit() if(!$dbh->{AutoCommit});}; + if ($@) { + $err = encode_base64($@,""); + Log3 ($name, 2, "DbRep $name - ERROR - $@"); + $dbh->disconnect; + return "$name|''|$opt|$sql|''|''|$err"; + } - push(@rows, $r); - my $com = (split(" ",$sql, 2))[0]; - Log3 ($name, 3, "DbRep $name - Number of entries processed in db $hash->{DATABASE}: $nrows by $com"); + push(@rows, $r); + my $com = (split(" ",$sql, 2))[0]; + Log3 ($name, 3, "DbRep $name - Number of entries processed in db $hash->{DATABASE}: $nrows by $com"); } $sth->finish; @@ -6294,8 +6333,8 @@ sub sqlCmd_ParseDone($) { foreach $row ( @rows ) { $row =~ s/\|°escaped°\|/§/g; $row =~ s/$srs/\|/g if($srs !~ /\|/); - $row =~ s/\|/<\/td>/g; - $res .= "".$row.""; + $row =~ s/\|/<\/td>/g; + $res .= "".$row.""; } $row .= $res.""; @@ -6461,6 +6500,7 @@ sub dbmeta_DoParse($) { } } } elsif ($opt eq "procinfo") { + my $row; my $res = ""; $res .= ""; $res .= ""; @@ -6471,11 +6511,12 @@ sub dbmeta_DoParse($) { $res .= ""; $res .= ""; $res .= ""; - while (my @line = $sth->fetchrow_array()) { + + while (my @line = $sth->fetchrow_array()) { Log3 ($name, 4, "DbRep $name - SQL result: @line"); - my $row = join("|", @line); - $row =~ tr/ A-Za-z0-9!"#$§%&'()*+,-.\/:;<=>?@[\]^_`{|}~//cd; - $row =~ s/\|/<\/td>"; } my $tab .= $res."
IDUSERSTATEINFOPROGRESS
/g; + $row = join("|", @line); + $row =~ tr/ A-Za-z0-9!"#$§%&'()*+,-.\/:;<=>?@[\]^_`{|}~//cd; + $row =~ s/\|/<\/td>/g; $res .= "
".$row."
"; @@ -9670,9 +9711,22 @@ sub DbRep_checktimeaggr ($) { my $IsAggrSet = 0; my $aggregation = AttrVal($name,"aggregation","no"); - if ( AttrVal($name,"timestamp_begin",undef) || AttrVal($name,"timestamp_end",undef) || - AttrVal($name,"timeDiffToNow",undef) || AttrVal($name,"timeOlderThan",undef) || AttrVal($name,"timeYearPeriod",undef) ) { - $IsTimeSet = 1; + my @a; + @a = @{$hash->{HELPER}{REDUCELOG}} if($hash->{HELPER}{REDUCELOG}); + @a = @{$hash->{HELPER}{DELENTRIES}} if($hash->{HELPER}{DELENTRIES}); + my $timeoption = 0; + foreach (@a) { # evtl. Relativzeiten bei "reduceLog" oder "deleteEntries" berücksichtigen + if($_ =~ /\b(\d+(:\d+)?)\b/) { + $timeoption = 1; + } + } + + if ( AttrVal($name,"timestamp_begin", undef) || + AttrVal($name,"timestamp_end", undef) || + AttrVal($name,"timeDiffToNow", undef) || + AttrVal($name,"timeOlderThan", undef) || + AttrVal($name,"timeYearPeriod", undef) || $timeoption ) { + $IsTimeSet = 1; } if ($aggregation ne "no") { @@ -9686,7 +9740,7 @@ sub DbRep_checktimeaggr ($) { $aggregation = "day"; # für Tagesmittelwertberechnung des deutschen Wetterdienstes immer "day" $IsAggrSet = 1; } - if($hash->{LASTCMD} =~ /delEntries|fetchrows|deviceRename|readingRename|tableCurrentFillup|reduceLog/) { + if($hash->{LASTCMD} =~ /delEntries|fetchrows|deviceRename|readingRename|tableCurrentFillup|reduceLog|\breadingsDifferenceByTimeDelta\b/) { $IsAggrSet = 0; $aggregation = "no"; } @@ -13010,12 +13064,16 @@ return;
@@ -15556,12 +15614,16 @@ sub bdump {
diff --git a/fhem/contrib/DS_Starter/93_DbRep.pm b/fhem/contrib/DS_Starter/93_DbRep.pm index 07337f8c4..ebd57d6f9 100644 --- a/fhem/contrib/DS_Starter/93_DbRep.pm +++ b/fhem/contrib/DS_Starter/93_DbRep.pm @@ -58,7 +58,8 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # Version History intern our %DbRep_vNotesIntern = ( - "8.38.0" => "21.03.2020 sqlSpecial readingsDifferenceByTimeDelta, fix FHEM crash if no time-attribute is set and time option of delEntries / reduceLog is used ", + "8.38.0" => "21.03.2020 sqlSpecial readingsDifferenceByTimeDelta, fix FHEM crash if no time-attribute is set and time ". + "option of delEntries / reduceLog is used, minor fix for sqlCmdHistory ", "8.37.0" => "20.03.2020 add column header for free selects (sqlCmd) ", "8.36.0" => "19.03.2020 sqlSpecial recentReadingsOfDevice ", "8.35.0" => "19.03.2020 option older than days and newer than days for delEntries function ", @@ -911,10 +912,13 @@ sub DbRep_Set($@) { # $sqlcmd =~ tr/ A-Za-z0-9!"#$§%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; # V8.36.0 20.03.2020 } if($opt eq "sqlCmdHistory") { - $prop =~ tr/ A-Za-z0-9!"#$%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; + $prop =~ s/§/_ESC_ECS_/g; + $prop =~ tr/ A-Za-z0-9!"#%&'()*+,-.\/:;<=>?@[\\]^_`{|}~äöüÄÖÜ߀/ /cs; + $prop =~ s/_ESC_ECS_/§/g; $prop =~ s//,/g; # noch aus Kompatibilitätsgründen enthalten $prop =~ s/(\x20)*\xbc/,/g; # Forum: https://forum.fhem.de/index.php/topic,103908.0.html $sqlcmd = $prop; + Log3 ($name, 1, "DbRep $name - com: $sqlcmd"); if($sqlcmd eq "___purge_historylist___") { delete($hash->{HELPER}{SQLHIST}); DbRep_setCmdFile($name."_sqlCmdList","",$hash); # Löschen der sql History Liste im DbRep-Keyfile