diff --git a/fhem/CHANGED b/fhem/CHANGED index a2ec683a9..57cd99d2b 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,7 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - feature: 93_DbRep: reduceLog - add max, max=day, min, min=day, sum, sum=day + some fixes of reduceLog routine - feature: 50_Signalbot: support to link 2 FHEM instance under the same number - feature: 93_DbLog: new Attr convertTimezone -> logging in UTC time - bugfix: 93_DbRep: fix set ... deleteOther for SQLite diff --git a/fhem/FHEM/93_DbRep.pm b/fhem/FHEM/93_DbRep.pm index baeadd130..4b00fd259 100644 --- a/fhem/FHEM/93_DbRep.pm +++ b/fhem/FHEM/93_DbRep.pm @@ -57,6 +57,7 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # Version History intern my %DbRep_vNotesIntern = ( + "8.50.0" => "20.08.2022 rework of DbRep_reduceLog - add max, max=day, min, min=day, sum, sum=day ", "8.49.1" => "03.08.2022 fix DbRep_deleteOtherFromDB, Forum: https://forum.fhem.de/index.php/topic,128605.0.html ". "some code changes and bug fixes ", "8.49.0" => "16.05.2022 allow optionally set device / reading in the insert command ", @@ -2925,12 +2926,11 @@ sub _DbRep_collaggstr { $runtime_orig = $runtime; # Hilfsrechnungen - my $rm = strftime "%m", localtime($runtime); # Monat des aktuell laufenden Startdatums d. SQL-Select - my $cy = strftime "%Y", localtime($runtime); # Jahr des aktuell laufenden Startdatums d. SQL-Select - # my $icly = DbRep_IsLeapYear($name,$cy); - # my $inly = DbRep_IsLeapYear($name,$cy+1); # ist kommendes Jahr ein Schaltjahr ? + my $rm = strftime "%m", localtime($runtime); # Monat des aktuell laufenden Startdatums d. SQL-Select + my $cy = strftime "%Y", localtime($runtime); # Jahr des aktuell laufenden Startdatums d. SQL-Select my $yf = 365; - $yf = 366 if(DbRep_IsLeapYear($name,$cy)); # ist aktuelles Jahr ein Schaltjahr ? + $yf = 366 if(DbRep_IsLeapYear($name,$cy)); # ist aktuelles Jahr ein Schaltjahr ? + Log3 ($name, 5, "DbRep $name - current year: $cy, endyear: $yestr"); $aggsec = $yf * 86400; @@ -2948,13 +2948,14 @@ sub _DbRep_collaggstr { $runtime_string_first = strftime "%Y-01-01", localtime($runtime) if($i>1); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - - } else { + } + else { if(($runtime) > $epoch_seconds_end) { $runtime_string_first = strftime "%Y-01-01", localtime($runtime); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_first = strftime "%Y-01-01", localtime($runtime) if($i>1); $runtime_string_next = strftime "%Y-01-01", localtime($runtime+($yf * 86400)); } @@ -2988,15 +2989,16 @@ sub _DbRep_collaggstr { if ($ysstr == $yestr && $msstr == $mestr || $ry == $yestr && $rm == $mestr) { $runtime_string_first = strftime "%Y-%m-01", localtime($runtime) if($i>1); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); - $ll=1; - - } else { + $ll=1; + } + else { if(($runtime) > $epoch_seconds_end) { #$runtime_string_first = strftime "%Y-%m-01", localtime($runtime) if($i>11); # ausgebaut 24.02.2018 $runtime_string_first = strftime "%Y-%m-01", localtime($runtime); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_first = strftime "%Y-%m-01", localtime($runtime) if($i>1); $runtime_string_next = strftime "%Y-%m-01", localtime($runtime+($dim*86400)); } @@ -3034,16 +3036,19 @@ sub _DbRep_collaggstr { if((strftime "%V", localtime($epoch_seconds_end)) eq ($w) && ($ms+$me != 13)) { $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_next = strftime "%Y-%m-%d", localtime($runtime); } - } else { + } + else { # weitere Durchläufe if(($runtime+$aggsec) > $epoch_seconds_end) { $runtime_string_first = strftime "%Y-%m-%d", localtime($runtime_orig); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_first = strftime "%Y-%m-%d", localtime($runtime_orig) ; $runtime_string_next = strftime "%Y-%m-%d", localtime($runtime+$aggsec); } @@ -3065,7 +3070,8 @@ sub _DbRep_collaggstr { $runtime_string_first = strftime "%Y-%m-%d %H:%M:%S", localtime($runtime) if( $dsstr eq $destr); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_next = strftime "%Y-%m-%d", localtime($runtime+$aggsec); } @@ -3094,7 +3100,8 @@ sub _DbRep_collaggstr { $runtime_string_first = strftime "%Y-%m-%d %H:%M:%S", localtime($runtime) if( $dsstr eq $destr && $hs eq $he); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_next = strftime "%Y-%m-%d %H", localtime($runtime+$aggsec); } @@ -3121,7 +3128,8 @@ sub _DbRep_collaggstr { # $runtime_string_first = strftime "%Y-%m-%d %H:%M", localtime($runtime) if( $dsstr eq $destr && $ms eq $me); $runtime_string_next = strftime "%Y-%m-%d %H:%M:%S", localtime($epoch_seconds_end); $ll=1; - } else { + } + else { $runtime_string_next = strftime "%Y-%m-%d %H:%M", localtime($runtime+$aggsec); } @@ -8946,11 +8954,6 @@ sub DbRep_reduceLog { } Log3 ($name, 5, "DbRep $name -> Start DbLog_reduceLog"); - - my ($dbh,$brt,$ret,$row,$filter,$exclude,$c,$day,$hour,$lastHour,$updDate,$updHour); - my ($dbmodel,$average,$processingDay,$lastUpdH); - my (%hourlyKnown,%averageHash,@excludeRegex,@dayRows,@averageUpd,@averageUpdD); - my ($startTime,$currentHour,$currentDay,$deletedCount,$updateCount,$sum,$rowCount,$excludeCount) = (time(),99,0,0,0,0,0,0); BlockingInformParent("DbRep_delHashValFromBlocking", [$name, "HELPER","REDUCELOG"], 1); @@ -8961,28 +8964,36 @@ sub DbRep_reduceLog { next if($w =~ /\b(\d+(:\d+)?)\b/); push @b, $w; } + @a = @b; my ($pa,$ph) = parseParams(join ' ', @a); - my $avgstring = (@$pa[1] && @$pa[1] =~ /average/i) ? 'AVERAGE=HOUR' : - ($ph->{average} && $ph->{average} eq "day") ? 'AVERAGE=DAY' : - q{}; - - # Korrektur des Select-Zeitraums + eine Stunde - # (Forum: https://forum.fhem.de/index.php/topic,53584.msg1177799.html#msg1177799) - my ($yyyy, $mm, $dd, $hh, $min, $sec) = $ots =~ /(\d+)-(\d+)-(\d+)\s(\d+):(\d+):(\d+)/x; - my $epoche = timelocal($sec, $min, $hh, $dd, $mm-1, $yyyy-1900); - my $splus = $avgstring =~ /AVERAGE/ ? 3600 : 0; - $ots = strftime "%Y-%m-%d %H:%M:%S", localtime($epoche+$splus); + my $mode = (@$pa[1] && @$pa[1] =~ /average/i) ? 'average' : + ($ph->{average} && $ph->{average} eq "day") ? 'average=day' : + (@$pa[1] && @$pa[1] =~ /max/i) ? 'max' : + ($ph->{max} && $ph->{max} eq "day") ? 'max=day' : + (@$pa[1] && @$pa[1] =~ /min/i) ? 'min' : + ($ph->{min} && $ph->{min} eq "day") ? 'min=day' : + (@$pa[1] && @$pa[1] =~ /sum/i) ? 'sum' : + ($ph->{sum} && $ph->{sum} eq "day") ? 'sum=day' : + q{}; + + my $mstr = $mode =~ /average/i ? 'average' : + $mode =~ /max/i ? 'max' : + $mode =~ /min/i ? 'min' : + $mode =~ /sum/i ? 'sum' : + q{}; my $excludes = $ph->{EXCLUDE} // q{}; my $includes = $ph->{INCLUDE} // q{}; + my @excludeRegex; if ($excludes) { - @excludeRegex = split(',',$excludes); + @excludeRegex = split ',', $excludes; } - + + my ($dbh, $dbmodel); ($err,$dbh,$dbmodel) = DbRep_dbConnect($name, 0); return "$name|$err" if ($err); @@ -8992,7 +9003,6 @@ sub DbRep_reduceLog { Log3 ($name, 5, "DbRep $name - IsTimeSet: $IsTimeSet, IsAggrSet: $IsAggrSet"); - my $sql; my $selspec = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM $table where "; my $addon = "ORDER BY TIMESTAMP ASC"; @@ -9008,6 +9018,8 @@ sub DbRep_reduceLog { addon => $addon }; + my $sql; + if($includes) { # Option EX/INCLUDE wurde angegeben $sql = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM $table WHERE " .($includes =~ /^(.+):(.+)$/i ? "DEVICE like '$1' AND READING like '$2' AND " : '') @@ -9018,19 +9030,16 @@ sub DbRep_reduceLog { elsif ($IsTimeSet || $IsAggrSet) { $specs->{rsf} = $nts; $specs->{rsn} = $ots; - $sql = DbRep_createCommonSql ($specs); + $sql = DbRep_createCommonSql ($specs); } else { $sql = DbRep_createCommonSql ($specs); } - - Log3 ($name, 3, "DbRep $name - reduce data older than: $ots (logical corrected), newer than: $nts"); - - Log3 ($name, 4, "DbRep $name - SQL execute: $sql"); + Log3 ($name, 3, "DbRep $name - reduce data older than: $ots, newer than: $nts"); Log3 ($name, 3, "DbRep $name - reduceLog requested with options: " - .($avgstring ? "\n".$avgstring : '') + .($mode ? "\n".$mode : '') .($includes ? "\nINCLUDE -> $includes " : ((($idanz || $idevswc || $iranz || $irdswc) ? "\nINCLUDE -> " : '') . (($idanz || $idevswc) ? "Devs: ".($idevs ? $idevs : '').($idevswc ? $idevswc : '').' ' : '').(($iranz || $irdswc) ? "Readings: ".($ireading ? $ireading : '').($irdswc ? $irdswc : '') : '') @@ -9043,377 +9052,793 @@ sub DbRep_reduceLog { my ($sth_del, $sth_upd, $sth_delD, $sth_updD, $sth_get); - eval { $sth_del = $dbh->prepare_cached("DELETE FROM $table WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); - $sth_upd = $dbh->prepare_cached("UPDATE $table SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); - $sth_delD = $dbh->prepare_cached("DELETE FROM $table WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); - $sth_updD = $dbh->prepare_cached("UPDATE $table SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); - $sth_get = $dbh->prepare($sql); - }; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - return "$name|$err"; - } + ($err, $sth_del) = DbRep_prepareCachedOnly ($name, $dbh, "DELETE FROM $table WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); + return "$name|$err" if ($err); - eval { $sth_get->execute(); }; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - return "$name|$err"; - } + ($err, $sth_upd) = DbRep_prepareCachedOnly ($name, $dbh, "UPDATE $table SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?) AND (VALUE=?)"); + return "$name|$err" if ($err); - ######################################### Start + ($err, $sth_delD) = DbRep_prepareCachedOnly ($name, $dbh, "DELETE FROM $table WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); + return "$name|$err" if ($err); - # Ergebnsis von $sth_get->fetchrow_arrayref + ($err, $sth_updD) = DbRep_prepareCachedOnly ($name, $dbh, "UPDATE $table SET TIMESTAMP=?, EVENT=?, VALUE=? WHERE (DEVICE=?) AND (READING=?) AND (TIMESTAMP=?)"); + return "$name|$err" if ($err); + + ($err, $sth_get) = DbRep_prepareExecuteQuery ($name, $dbh, $sql); + return "$name|$err" if ($err); + + + ## Start + ############################################ + + # Ergebnis von $sth_get->fetchrow_arrayref: # $row->[0] = Datum (YYYY-MM-DD hh:mm:ss) # $row->[1] = Device # $row->[2] = leer # $row->[3] = Reading # $row->[4] = Value + my $ndp = AttrVal($name, "numDecimalPlaces", $dbrep_defdecplaces); + + my ($day, $hour, $processingDay, $params); + my (%hourlyKnown,@dayRows,@updateHour,@updateDay); + my ($startTime,$currentHour,$currentDay,$deletedCount,$updateCount,$rowCount,$excludeCount) = (time(),99,0,0,0,0,0); + do { - $row = $sth_get->fetchrow_arrayref || ['0000-00-00 00:00:00','D','','R','V']; # || execute last-day dummy - $ret = 1; - ($day,$hour) = $row->[0] =~ /-(\d{2})\s(\d{2}):/; + my $row = $sth_get->fetchrow_arrayref || ['0000-00-00 00:00:00','D','','R','V']; # || execute last-day dummy + my $ts = $row->[0]; + my $device = $row->[1]; + my $reading = $row->[3]; + my $value = $row->[4]; + ($day,$hour) = $ts =~ /-(\d{2})\s(\d{2}):/; $rowCount++ if($day != 00); + ## verarbeiten der unten vorbereiteten Arrays und Hashes + ######################################################### + if ($day != $currentDay) { - if ($currentDay) { # nicht am ersten ausgeführten Tag - if (scalar @dayRows) { - ($lastHour) = $dayRows[-1]->[0] =~ /(.*\d+\s\d{2}):/; - $c = 0; + if ($currentDay) { # nicht am ersten ausgeführten Tag + if (scalar @dayRows) { # alle Tageseinträge löschen - for my $delRow (@dayRows) { - $c++ if($day != 00 || $delRow->[0] !~ /$lastHour/); - } + $params = { + name => $name, + dbh => $dbh, + sth_del => $sth_del, + table => $table, + dayRowsref => \@dayRows, + deletedCountref => \$deletedCount, + processingDay => $processingDay, + ndp => $ndp + }; + + $err = _DbRep_rl_deleteDayRows ($params); + return "$name|$err" if ($err); - if($c) { - $deletedCount += $c; - - Log3 ($name, 3, "DbRep $name - reduceLog deleting $c records of day: $processingDay"); - - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - eval {$dbh->begin_work() if($dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - - eval { - my $i = 0; - my $k = 1; - my $th = ($#dayRows <= 2000) ? 100 : - ($#dayRows <= 30000) ? 1000 : - 10000; - - for my $delRow (@dayRows) { - if($day != 00 || $delRow->[0] !~ /$lastHour/) { - - Log3 ($name, 5, "DbRep $name - DELETE FROM $table WHERE (DEVICE=$delRow->[1]) AND (READING=$delRow->[3]) AND (TIMESTAMP=$delRow->[0]) AND (VALUE=$delRow->[4])"); - - $sth_del->execute(($delRow->[1], $delRow->[3], $delRow->[0], $delRow->[4])); - $i++; - - if($i == $th) { - my $prog = $k * $i; - - Log3 ($name, 3, "DbRep $name - reduceLog deletion progress of day: $processingDay is: $prog"); - - $i = 0; - $k++; - } - } - } - }; - - if ($@) { - $err = encode_base64($@, ""); - - Log3 ($name, 2, "DbRep $name - reduceLog ! FAILED ! for day $processingDay: $@"); - - eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - $ret = 0; - } - else { - eval {$dbh->commit() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - } - $dbh->{RaiseError} = 0; - $dbh->{PrintError} = 1; - } @dayRows = (); } - if ($ret && $avgstring =~ /average/i) { - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - eval {$dbh->begin_work() if($dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - - eval { - push(@averageUpd, {%hourlyKnown}) if($day != 00); - - $c = 0; - for my $hourHash (@averageUpd) { # Only count for logging... - for my $hourKey (keys %$hourHash) { - $c++ if ($hourHash->{$hourKey}->[0] && scalar(@{$hourHash->{$hourKey}->[4]}) > 1); - } - } - - $updateCount += $c; - - Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating $c records of day: $processingDay") if($c); # else only push to @averageUpdD - - my $i = 0; - my $k = 1; - my $th = $c <= 2000 ? 100 : - $c <= 30000 ? 1000 : - 10000; - - for my $hourHash (@averageUpd) { - for my $hourKey (keys %$hourHash) { - if ($hourHash->{$hourKey}->[0]) { # true if reading is a number - ($updDate,$updHour) = $hourHash->{$hourKey}->[0] =~ /(.*\d+)\s(\d{2}):/; - - if (scalar(@{$hourHash->{$hourKey}->[4]}) > 1) { # true if reading has multiple records this hour - - for (@{$hourHash->{$hourKey}->[4]}) { - $sum += $_; - } - - $average = sprintf('%.3f', $sum/scalar(@{$hourHash->{$hourKey}->[4]}) ); - $sum = 0; - - Log3 ($name, 4, "DbRep $name - UPDATE $table SET TIMESTAMP=$updDate $updHour:30:00, EVENT='rl_av_h', VALUE=$average WHERE DEVICE=$hourHash->{$hourKey}->[1] AND READING=$hourHash->{$hourKey}->[3] AND TIMESTAMP=$hourHash->{$hourKey}->[0] AND VALUE=$hourHash->{$hourKey}->[4]->[0]"); - - $sth_upd->execute(("$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[4]->[0])); - - $i++; - - if($i == $th) { - my $prog = $k * $i; - Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating progress of day: $processingDay is: $prog"); - $i = 0; - $k++; - } - - if ($avgstring =~ /average=day/i) { - push(@averageUpdD, ["$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]); - } - } - else { - if ($avgstring =~ /average=day/i) { - push(@averageUpdD, [$hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[2], $hourHash->{$hourKey}->[4]->[0], $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]); - } - } - } - } - } + if ($mode =~ /average|max|min|sum/i) { + + $params = { + name => $name, + dbh => $dbh, + sth_upd => $sth_upd, + mode => $mode, + mstr => $mstr, + table => $table, + hourlyKnownref => \%hourlyKnown, + updateHourref => \@updateHour, + updateDayref => \@updateDay, + updateCountref => \$updateCount, + processingDay => $processingDay, + ndp => $ndp }; + + $err = _DbRep_rl_updateHour ($params); + return "$name|$err" if ($err); - if ($@) { - $err = $@; - - Log3 ($name, 2, "DbRep $name - reduceLog average=hour ! FAILED ! for day $processingDay: $err"); - - eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - @averageUpdD = (); - } - else { - eval {$dbh->commit() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - } - - $dbh->{RaiseError} = 0; - $dbh->{PrintError} = 1; - @averageUpd = (); + @updateHour = (); } - if ($avgstring =~ /average=day/i && scalar(@averageUpdD) && $day != 00) { - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - eval {$dbh->begin_work() if($dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - - eval { - for (@averageUpdD) { - push(@{$averageHash{$_->[3].$_->[4]}->{tedr}}, [$_->[0], $_->[1], $_->[3], $_->[4]]); - $averageHash{$_->[3].$_->[4]}->{sum} += $_->[2]; - $averageHash{$_->[3].$_->[4]}->{date} = $_->[5]; - } - - $c = 0; - for (keys %averageHash) { - - if(scalar @{$averageHash{$_}->{tedr}} == 1) { - delete $averageHash{$_}; - } - else { - $c += (scalar(@{$averageHash{$_}->{tedr}}) - 1); - } - } - $deletedCount += $c; - $updateCount += keys(%averageHash); - - my ($id,$iu) = (0,0); - my ($kd,$ku) = (1,1); - my $thd = ($c <= 2000) ? 100 : - ($c <= 30000) ? 1000 : - 10000; - my $thu = ((keys %averageHash) <= 2000) ? 100 : - ((keys %averageHash) <= 30000) ? 1000 : - 10000; - - Log3 ($name, 3, "DbRep $name - reduceLog (daily-average) updating ".(keys %averageHash).", deleting $c records of day: $processingDay") if(keys %averageHash); - - for my $reading (keys %averageHash) { - $average = sprintf('%.3f', $averageHash{$reading}->{sum}/scalar(@{$averageHash{$reading}->{tedr}})); - $lastUpdH = pop @{$averageHash{$reading}->{tedr}}; - - for (@{$averageHash{$reading}->{tedr}}) { - Log3 ($name, 5, "DbRep $name - DELETE FROM $table WHERE DEVICE='$_->[2]' AND READING='$_->[3]' AND TIMESTAMP='$_->[0]'"); - - $sth_delD->execute(($_->[2], $_->[3], $_->[0])); - - $id++; - if($id == $thd) { - my $prog = $kd * $id; - Log3 ($name, 3, "DbRep $name - reduceLog (daily-average) deleting progress of day: $processingDay is: $prog"); - $id = 0; - $kd++; - } - } - - Log3 ($name, 4, "DbRep $name - UPDATE $table SET TIMESTAMP=$averageHash{$reading}->{date} 12:00:00, EVENT='rl_av_d', VALUE=$average WHERE (DEVICE=$lastUpdH->[2]) AND (READING=$lastUpdH->[3]) AND (TIMESTAMP=$lastUpdH->[0])"); - - $sth_updD->execute(($averageHash{$reading}->{date}." 12:00:00", 'rl_av_d', $average, $lastUpdH->[2], $lastUpdH->[3], $lastUpdH->[0])); - - $iu++; - if($iu == $thu) { - my $prog = $ku * $id; - Log3 ($name, 3, "DbRep $name - reduceLog (daily-average) updating progress of day: $processingDay is: $prog"); - $iu = 0; - $ku++; - } - } + if ($mode =~ /=day/i && scalar @updateDay) { + + $params = { + name => $name, + dbh => $dbh, + sth_delD => $sth_delD, + sth_updD => $sth_updD, + mode => $mode, + mstr => $mstr, + table => $table, + updateDayref => \@updateDay, + deletedCountref => \$deletedCount, + updateCountref => \$updateCount, + processingDay => $processingDay, + ndp => $ndp }; - - if ($@) { - Log3 ($name, 3, "DbRep $name - reduceLog average=day ! FAILED ! for day $processingDay"); - eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - } - else { - eval {$dbh->commit() if(!$dbh->{AutoCommit});}; - if ($@) { - $err = encode_base64($@, ""); - Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); - } - } - - $dbh->{RaiseError} = 0; - $dbh->{PrintError} = 1; + + $err = _DbRep_rl_updateDay ($params); + return "$name|$err" if ($err); + } - - %averageHash = (); + %hourlyKnown = (); - @averageUpd = (); - @averageUpdD = (); + @updateHour = (); + @updateDay = (); $currentHour = 99; } $currentDay = $day; } + ## Füllen Arrays und Hashes + ############################ + if ($hour != $currentHour) { # forget records from last hour, but remember these for average - if ($avgstring =~ /average/i && keys(%hourlyKnown)) { - push(@averageUpd, {%hourlyKnown}); + if ($mode =~ /average|max|min|sum/i && keys(%hourlyKnown)) { + push(@updateHour, {%hourlyKnown}); } %hourlyKnown = (); $currentHour = $hour; } - - if (defined $hourlyKnown{$row->[1].$row->[3]}) { # remember first readings for device per h, other can be deleted + + if (defined $hourlyKnown{$device.$reading}) { # das erste reading pro device und Stunde wird nicht in @dayRows (zum Löschen) gespeichert, die anderen können gelöscht werden push(@dayRows, [@$row]); - if ($avgstring =~ /average/i && - defined($row->[4]) && - $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/ && - $hourlyKnown{$row->[1].$row->[3]}->[0]) { + if ($mode =~ /average|max|min|sum/i && + defined($value) && + DbRep_IsNumeric ($value) && + $hourlyKnown{$device.$reading}->[0]) { - if ($hourlyKnown{$row->[1].$row->[3]}->[0]) { - push(@{$hourlyKnown{$row->[1].$row->[3]}->[4]}, $row->[4]); + if ($hourlyKnown{$device.$reading}->[0]) { + push(@{$hourlyKnown{$device.$reading}->[4]}, $value); } } } else { - $exclude = 0; + my $exclude = 0; - for (@excludeRegex) { - $exclude = 1 if("$row->[1]:$row->[3]" =~ /^$_$/); + for my $exreg (@excludeRegex) { + $exclude = 1 if("$device:$reading" =~ /^$exreg$/); } if ($exclude) { $excludeCount++ if($day != 00); } else { - $hourlyKnown{$row->[1].$row->[3]} = (defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/) ? [$row->[0],$row->[1],$row->[2],$row->[3],[$row->[4]]] : [0]; + $hourlyKnown{$device.$reading} = DbRep_IsNumeric ($value) ? + [$ts,$device,$row->[2],$reading,[$value]] : + [0]; } } - $processingDay = (split(' ',$row->[0]))[0]; + + $processingDay = (split ' ', $ts)[0]; # $ts = Datum (YYYY-MM-DD hh:mm:ss) - } while ($day != 00); # die do...while-Anweisung überprüft die Bedingung am Ende jeder Iteration. + } while ($day != 00); # die do...while-Anweisung überprüft die Bedingung am Ende jeder Iteration. ######################################### Ende - $brt = time() - $startTime; + my $brt = time() - $startTime; my $result = "Rows processed: $rowCount, deleted: $deletedCount" - .($avgstring =~ /average/i ? ", updated: $updateCount" : '') - .($excludeCount ? ", excluded: $excludeCount" : ''); + .($mode =~ /average|max|min|sum/i ? ", updated: $updateCount" : '') + .($excludeCount ? ", excluded: $excludeCount" : ''); Log3 ($name, 3, "DbRep $name - reduceLog finished. $result"); - $ret = "reduceLog finished. $result"; - $dbh->disconnect(); - $ret = encode_base64($ret, ""); + my $ret = encode_base64("reduceLog finished. $result", ""); Log3 ($name, 5, "DbRep $name -> DbRep_reduceLogNbl finished"); return "$name|$err|$ret|$brt"; } +#################################################################################################### +# reduceLog alle im @dayRows Array enthaltene DB Einträge löschen +#################################################################################################### +sub _DbRep_rl_deleteDayRows { + my $paref = shift; + my $name = $paref->{name}; + my $dbh = $paref->{dbh}; + my $sth_del = $paref->{sth_del}; + my $table = $paref->{table}; + my $dayRowsref = $paref->{dayRowsref}; + my $deletedCountref = $paref->{deletedCountref}; + my $processingDay = $paref->{processingDay}; + + my $err = q{}; + + my @dayRows = @{$dayRowsref}; + + #Log3 ($name, 3, "DbRep $name - content dayRows Array:\n".Dumper @dayRows); + + my $c = 0; + + for my $delRow (@dayRows) { + $c++; + } + + if($c) { + ${$deletedCountref} += $c; + + Log3 ($name, 3, "DbRep $name - reduceLog deleting $c records of day: $processingDay"); + + $err = DbRep_beginDatabaseTransaction ($name, $dbh); + return $err if ($err); + + eval { + my $i = 0; + my $k = 1; + my $th = _DbRep_rl_logThreshold ($#dayRows); + + for my $delRow (@dayRows) { + Log3 ($name, 5, "DbRep $name - DELETE FROM $table WHERE (DEVICE=$delRow->[1]) AND (READING=$delRow->[3]) AND (TIMESTAMP=$delRow->[0]) AND (VALUE=$delRow->[4])"); + + $sth_del->execute(($delRow->[1], $delRow->[3], $delRow->[0], $delRow->[4])); + $i++; + + my $params = { + name => $name, + logtxt => "deletion", + iref => \$i, + kref => \$k, + th => $th, + processingDay => $processingDay + }; + + _DbRep_rl_logProgress ($params); + } + 1; + } + or do { + $err = encode_base64($@, ""); + + Log3 ($name, 2, "DbRep $name - ERROR - reduceLog failed for day $processingDay: $@"); + + DbRep_rollbackOnly ($name, $dbh); + return $err; + }; + + $err = DbRep_commitOnly ($name, $dbh); + return $err if ($err); + } + +return $err; +} + +#################################################################################################### +# reduceLog +# Stundenupdates vornehmen und @updateDay füllen bei +# $mode = *=day +#################################################################################################### +sub _DbRep_rl_updateHour { + my $paref = shift; + my $name = $paref->{name}; + my $dbh = $paref->{dbh}; + my $sth_upd = $paref->{sth_upd}; + my $mode = $paref->{mode}; + my $mstr = $paref->{mstr}; + my $table = $paref->{table}; + my $hourlyKnownref = $paref->{hourlyKnownref}; + my $updateHourref = $paref->{updateHourref}; + my $updateDayref = $paref->{updateDayref}; + my $updateCountref = $paref->{updateCountref}; + my $processingDay = $paref->{processingDay}; + my $ndp = $paref->{ndp}; + + my $err = q{}; + + #Log3 ($name, 3, "DbRep $name - content hourlyKnown Hash:\n".Dumper %$hourlyKnownref); + + push(@$updateHourref, {%$hourlyKnownref}); + + my $c = 0; + + for my $hourHash (@$updateHourref) { # Only count for logging... + for my $hourKey (keys %$hourHash) { + $c++ if ($hourHash->{$hourKey}->[0] && scalar @{$hourHash->{$hourKey}->[4]} > 1); + } + } + + ${$updateCountref} += $c; + + if($c) { + Log3 ($name, 3, "DbRep $name - reduceLog (hourly-$mstr) updating $c records of day: $processingDay"); + + $err = DbRep_beginDatabaseTransaction ($name, $dbh); + return $err if ($err); + } + + my ($params, $value); + my $i = 0; + my $k = 1; + my $th = _DbRep_rl_logThreshold ($c); + + my $event = $mstr eq 'average' ? 'rl_av_h' : + $mstr eq 'max' ? 'rl_max_h' : + $mstr eq 'min' ? 'rl_min_h' : + $mstr eq 'sum' ? 'rl_sum_h' : + 'rl_h'; + + my $updminutes = $mstr eq 'average' ? '30:00' : + $mstr eq 'max' ? '59:59' : + $mstr eq 'min' ? '00:01' : + $mstr eq 'sum' ? '00:00' : + '00:00'; + + #Log3 ($name, 3, "DbRep $name - content updateHour Array:\n".Dumper @$updateHourref); + + $paref->{updminutes} = $updminutes; + $paref->{event} = $event; + $paref->{th} = $th; + $paref->{iref} = \$i; + $paref->{kref} = \$k; + + for my $hourHash (@$updateHourref) { + + for my $hourKey (keys %$hourHash) { + + next if (!$hourHash->{$hourKey}->[0]); + my ($updDate,$updHour) = $hourHash->{$hourKey}->[0] =~ /(.*\d+)\s(\d{2}):/; + + $paref->{updDate} = $updDate; + $paref->{updHour} = $updHour; + $paref->{timestamp} = $hourHash->{$hourKey}->[0]; + $paref->{device} = $hourHash->{$hourKey}->[1]; + $paref->{reading} = $hourHash->{$hourKey}->[3]; + $paref->{oldvalue} = $hourHash->{$hourKey}->[4]->[0]; + + if (scalar @{$hourHash->{$hourKey}->[4]} > 1) { # wahr wenn reading hat mehrere Datensätze diese Stunde + + $i++; + + $paref->{hourHashKeyRef} = $hourHash->{$hourKey}->[4]; + + if ($mstr eq 'average') { # Berechnung Average + $value = __DbRep_rl_calcAverageHourly ($paref); + } + elsif ($mstr eq 'max') { # Berechnung Max + $value = __DbRep_rl_calcMaxHourly ($paref); + } + elsif ($mstr eq 'min') { # Berechnung Min + $value = __DbRep_rl_calcMinHourly ($paref); + } + elsif ($mstr eq 'sum') { # Berechnung Summary + $value = __DbRep_rl_calcSumHourly ($paref); + } + + $paref->{logtxt} = "(hourly-$mstr) updating"; + $paref->{newvalue} = $value; + + $err = __DbRep_rl_updateHourDatabase ($paref); + + if ($err) { + Log3 ($name, 2, "DbRep $name - ERROR - reduceLog $mstr failed for day $processingDay: $err"); + $err = encode_base64($err, ""); + + DbRep_rollbackOnly ($name, $dbh); + return $err; + } + } + else { + __DbRep_rl_onlyFillDayArray ($paref); + } + } + } + + if($c) { + $err = DbRep_commitOnly ($name, $dbh); + return $err if ($err); + } + +return $err; +} + +#################################################################################################### +# reduceLog stündlichen average Wert berechnen +#################################################################################################### +sub __DbRep_rl_calcAverageHourly { + my $paref = shift; + my $name = $paref->{name}; + my $hourHashKeyRef = $paref->{hourHashKeyRef}; + my $ndp = $paref->{ndp}; + + my $sum = 0; + + for my $val (@{$hourHashKeyRef}) { + $sum += $val; + } + + my $value = sprintf "%.${ndp}f", $sum / scalar @{$hourHashKeyRef}; + +return $value; +} + +#################################################################################################### +# reduceLog stündlichen Max Wert berechnen +#################################################################################################### +sub __DbRep_rl_calcMaxHourly { + my $paref = shift; + my $name = $paref->{name}; + my $hourHashKeyRef = $paref->{hourHashKeyRef}; + my $ndp = $paref->{ndp}; + + my $max; + + for my $val (@{$hourHashKeyRef}) { + if (!defined $max) { + $max = $val; + } + else { + $max = $val if ($val > $max); + } + } + + my $value = sprintf "%.${ndp}f", $max; + +return $value; +} + +#################################################################################################### +# reduceLog stündlichen Min Wert berechnen +#################################################################################################### +sub __DbRep_rl_calcMinHourly { + my $paref = shift; + my $name = $paref->{name}; + my $hourHashKeyRef = $paref->{hourHashKeyRef}; + my $ndp = $paref->{ndp}; + + my $min; + + for my $val (@{$hourHashKeyRef}) { + if (!defined $min) { + $min = $val; + } + else { + $min = $val if ($val < $min); + } + } + + my $value = sprintf "%.${ndp}f", $min; + +return $value; +} + +#################################################################################################### +# reduceLog stündlichen summary Wert berechnen +#################################################################################################### +sub __DbRep_rl_calcSumHourly { + my $paref = shift; + my $name = $paref->{name}; + my $hourHashKeyRef = $paref->{hourHashKeyRef}; + my $ndp = $paref->{ndp}; + + my $sum = 0; + + for my $val (@{$hourHashKeyRef}) { + $sum += $val; + } + + my $value = sprintf "%.${ndp}f", $sum; + +return $value; +} + +################################################################ +# reduceLog Stundenupdate Datenbank und +# füllen Tages Update Array +################################################################ +sub __DbRep_rl_updateHourDatabase { + my $paref = shift; + my $name = $paref->{name}; + my $mode = $paref->{mode}; + my $table = $paref->{table}; + my $sth_upd = $paref->{sth_upd}; + my $updateDayref = $paref->{updateDayref}; + my $updDate = $paref->{updDate}; + my $updHour = $paref->{updHour}; + my $updminutes = $paref->{updminutes}; + my $event = $paref->{event}; + my $newvalue = $paref->{newvalue}; + my $device = $paref->{device}; + my $reading = $paref->{reading}; + my $timestamp = $paref->{timestamp}; + my $oldvalue = $paref->{oldvalue}; + + Log3 ($name, 4, "DbRep $name - UPDATE $table SET TIMESTAMP=$updDate $updHour:$updminutes, EVENT=$event, VALUE=$newvalue WHERE DEVICE=$device AND READING=$reading AND TIMESTAMP=$timestamp AND VALUE=$oldvalue"); + + eval { $sth_upd->execute("$updDate $updHour:$updminutes", $event, $newvalue, $device, $reading, $timestamp, $oldvalue); + } + or do { return $@; + }; + + _DbRep_rl_logProgress ($paref); + + if ($mode =~ /=day/i) { + push(@$updateDayref, ["$updDate $updHour:$updminutes", $event, $newvalue, $device, $reading, $updDate]); + } + +return; +} + +################################################################ +# reduceLog Tages Array füllen +################################################################ +sub __DbRep_rl_onlyFillDayArray { + my $paref = shift; + my $mode = $paref->{mode}; + my $updateDayref = $paref->{updateDayref}; + my $timestamp = $paref->{timestamp}; + my $event = $paref->{event}; + my $oldvalue = $paref->{oldvalue}; + my $device = $paref->{device}; + my $reading = $paref->{reading}; + my $updDate = $paref->{updDate}; + + if ($mode =~ /=day/i) { + push(@$updateDayref, [$timestamp, $event, $oldvalue, $device, $reading, $updDate]); + } + +return; +} + +#################################################################################################### +# reduceLog Tagesupdates vornehmen +#################################################################################################### +sub _DbRep_rl_updateDay { + my $paref = shift; + my $name = $paref->{name}; + my $dbh = $paref->{dbh}; + my $sth_delD = $paref->{sth_delD}; + my $sth_updD = $paref->{sth_updD}; + my $mode = $paref->{mode}; + my $mstr = $paref->{mstr}; + my $table = $paref->{table}; + my $updateDayref = $paref->{updateDayref}; + my $deletedCountref = $paref->{deletedCountref}; + my $updateCountref = $paref->{updateCountref}; + my $processingDay = $paref->{processingDay}; + my $ndp = $paref->{ndp}; + + my $err = q{}; + + #Log3 ($name, 3, "DbRep $name - content updateDay Array:\n".Dumper @$updateDayref); + + my %updateHash; + + for my $row (@$updateDayref) { + $updateHash{$row->[3].$row->[4]}->{date} = $row->[5]; + push @{$updateHash{$row->[3].$row->[4]}->{tedr}}, [$row->[0], $row->[1], $row->[3], $row->[4]]; # tedr -> time, event, device, reading + + if ($mstr eq 'average') { # Day Average + $updateHash{$row->[3].$row->[4]}->{sum} += $row->[2]; # Summe aller Werte + } + elsif ($mstr eq 'max') { # Day Max + if (!defined $updateHash{$row->[3].$row->[4]}->{max}) { + $updateHash{$row->[3].$row->[4]}->{max} = $row->[2]; + } + else { + $updateHash{$row->[3].$row->[4]}->{max} = $row->[2] if ($row->[2] > $updateHash{$row->[3].$row->[4]}->{max}); + } + } + elsif ($mstr eq 'min') { # Day Min + if (!defined $updateHash{$row->[3].$row->[4]}->{min}) { + $updateHash{$row->[3].$row->[4]}->{min} = $row->[2]; + } + else { + $updateHash{$row->[3].$row->[4]}->{min} = $row->[2] if ($row->[2] < $updateHash{$row->[3].$row->[4]}->{min}); + } + } + elsif ($mstr eq 'sum') { # Day Summary + $updateHash{$row->[3].$row->[4]}->{sum} += $row->[2]; # Summe aller Werte + } + } + + my $c = 0; + + for my $key (keys %updateHash) { + if(scalar @{$updateHash{$key}->{tedr}} == 1) { + delete $updateHash{$key}; + } + else { + $c += (scalar @{$updateHash{$key}->{tedr}} - 1); + } + } + + ${$deletedCountref} += $c; + ${$updateCountref} += keys %updateHash; + + my ($params, $value); + my ($id,$iu) = (0,0); + my ($kd,$ku) = (1,1); + my $thd = _DbRep_rl_logThreshold ($c); + my $thu = _DbRep_rl_logThreshold (scalar keys %updateHash); + + my $event = $mstr eq 'average' ? 'rl_av_d' : + $mstr eq 'max' ? 'rl_max_d' : + $mstr eq 'min' ? 'rl_min_d' : + $mstr eq 'sum' ? 'rl_sum_d' : + 'rl_d'; + + my $time = $mstr eq 'average' ? '12:00:00' : + $mstr eq 'max' ? '23:59:59' : + $mstr eq 'min' ? '00:00:01' : + $mstr eq 'sum' ? '12:00:00' : + '00:00:00'; + + $paref->{time} = $time; + $paref->{event} = $event; + + if(keys %updateHash) { + Log3 ($name, 3, "DbRep $name - reduceLog (daily-$mstr) updating ".(keys %updateHash).", deleting $c records of day: $processingDay"); + + $err = DbRep_beginDatabaseTransaction ($name, $dbh); + return $err if ($err); + } + + for my $uhk (keys %updateHash) { + + if ($mstr eq 'average') { # Day Average + $value = sprintf "%.${ndp}f", $updateHash{$uhk}->{sum} / scalar @{$updateHash{$uhk}->{tedr}}; + } + elsif ($mstr eq 'max') { # Day Max + $value = sprintf "%.${ndp}f", $updateHash{$uhk}->{max}; + } + elsif ($mstr eq 'min') { # Day Min + $value = sprintf "%.${ndp}f", $updateHash{$uhk}->{min}; + } + elsif ($mstr eq 'sum') { # Day Summary + $value = sprintf "%.${ndp}f", $updateHash{$uhk}->{sum}; + } + + my $lastUpdH = pop @{$updateHash{$uhk}->{tedr}}; + + for my $tedr (@{$updateHash{$uhk}->{tedr}}) { + $id++; + + $paref->{logtxt} = "(daily-$mstr) deleting"; + $paref->{iref} = \$id; + $paref->{kref} = \$kd; + $paref->{th} = $thd; + $paref->{timestamp} = $tedr->[0]; + $paref->{device} = $tedr->[2]; + $paref->{reading} = $tedr->[3]; + + $err = __DbRep_rl_deleteDayDatabase ($paref); + + if ($err) { + Log3 ($name, 3, "DbRep $name - ERROR - reduceLog $mstr=day failed for day $processingDay: $err"); + $err = encode_base64($err, ""); + + DbRep_rollbackOnly ($name, $dbh); + return $err; + } + } + + $iu++; + + $paref->{logtxt} = "(daily-$mstr) updating"; + $paref->{iref} = \$iu; + $paref->{kref} = \$ku; + $paref->{th} = $thu; + $paref->{date} = $updateHash{$uhk}->{date}; + $paref->{timestamp} = $lastUpdH->[0]; + $paref->{device} = $lastUpdH->[2]; + $paref->{reading} = $lastUpdH->[3]; + $paref->{value} = $value; + + $err = __DbRep_rl_updateDayDatabase ($paref); + + if ($err) { + Log3 ($name, 3, "DbRep $name - ERROR - reduceLog $mstr=day failed for day $processingDay: $err"); + $err = encode_base64($err, ""); + + DbRep_rollbackOnly ($name, $dbh); + return $err; + } + } + + if(keys %updateHash) { + $err = DbRep_commitOnly ($name, $dbh); + return $err if ($err); + } + +return $err; +} + +################################################################ +# reduceLog Tageswerte löschen +################################################################ +sub __DbRep_rl_deleteDayDatabase { + my $paref = shift; + my $name = $paref->{name}; + my $table = $paref->{table}; + my $sth_delD = $paref->{sth_delD}; + my $device = $paref->{device}; + my $reading = $paref->{reading}; + my $timestamp = $paref->{timestamp}; + + Log3 ($name, 4, "DbRep $name - DELETE FROM $table WHERE DEVICE='$device' AND READING='$reading' AND TIMESTAMP='$timestamp'"); + + eval { $sth_delD->execute($device, $reading, $timestamp); + } + or do { return $@; + }; + + _DbRep_rl_logProgress ($paref); + +return; +} + +################################################################ +# reduceLog Tageswerte updaten +################################################################ +sub __DbRep_rl_updateDayDatabase { + my $paref = shift; + my $name = $paref->{name}; + my $table = $paref->{table}; + my $sth_updD = $paref->{sth_updD}; + my $event = $paref->{event}; + my $device = $paref->{device}; + my $reading = $paref->{reading}; + my $value = $paref->{value}; + my $date = $paref->{date}; + my $time = $paref->{time}; + my $timestamp = $paref->{timestamp}; + + Log3 ($name, 4, "DbRep $name - UPDATE $table SET TIMESTAMP=$date $time, EVENT=$event, VALUE=$value WHERE (DEVICE=$device) AND (READING=$reading) AND (TIMESTAMP=$timestamp)"); + + eval { $sth_updD->execute("$date $time", $event, $value, $device, $reading, $timestamp); + } + or do { return $@; + }; + + _DbRep_rl_logProgress ($paref); + +return; +} + +#################################################################################################### +# reduceLog Grenzen für Logausgabe abhängig von der Zeilenanzahl +#################################################################################################### +sub _DbRep_rl_logThreshold { + my $rn = shift; + + my $th = ($rn <= 2000) ? 100 : + ($rn <= 30000) ? 1000 : + 10000; + +return $th; +} + +################################################################ +# reduceLog Logausgabe Fortschritt +################################################################ +sub _DbRep_rl_logProgress { + my $paref = shift; + my $name = $paref->{name}; + my $logtxt = $paref->{logtxt}; + my $iref = $paref->{iref}; + my $kref = $paref->{kref}; + my $th = $paref->{th}; + my $processingDay = $paref->{processingDay}; + + if(${$iref} == $th) { + my $prog = ${$kref} * ${$iref}; + + Log3 ($name, 3, "DbRep $name - reduceLog $logtxt progress of day: $processingDay is: $prog"); + + ${$iref} = 0; + ${$kref}++; + } + +return; +} + #################################################################################################### # reduceLog non-blocking Rückkehrfunktion #################################################################################################### @@ -10323,6 +10748,32 @@ sub DbRep_prepareOnly { return ($err, $sth); } +#################################################################################################### +# nur SQL prepare Cached +# return $sth bei Erfolg +#################################################################################################### +sub DbRep_prepareCachedOnly { + my $name = shift; + my $dbh = shift; + my $sql = shift; + my $info = shift // "SQL prepare cached: $sql"; + + my $err = q{}; + + my $sth; + + Log3 ($name, 4, "DbRep $name - $info"); + + eval{ $sth = $dbh->prepare_cached($sql); + } + or do { $err = encode_base64($@,""); + Log3 ($name, 2, "DbRep $name - ERROR - $@"); + $dbh->disconnect; + }; + +return ($err, $sth); +} + #################################################################################################### # SQL Query evaluieren und Return-String (bei Error in Verarbeitung) und $sth-String # bei Erfolg @@ -10431,6 +10882,34 @@ sub DbRep_commitOnly { return $err; } +#################################################################################################### +# nur Datenbank "rollback" +#################################################################################################### +sub DbRep_rollbackOnly { + my $name = shift; + my $dbh = shift; + my $info = shift // "transaction rollback"; + + my $err = q{}; + + eval{ if(!$dbh->{AutoCommit}) { + $dbh->rollback(); + Log3 ($name, 4, "DbRep $name - $info"); + 1; + } + else { + Log3 ($name, 4, "DbRep $name - data auto rollback"); + 1; + } + } + or do { $err = encode_base64($@,""); + Log3 ($name, 2, "DbRep $name - ERROR - $@"); + $dbh->disconnect; + }; + +return $err; +} + #################################################################################################### # Whitespace am Anfang / Ende eines Strings entfernen #################################################################################################### @@ -10812,7 +11291,7 @@ sub DbRep_normRelTime { $fdopt = ($aval =~ /FullDay/x && $toth >= 86400) ? 1 : 0; # ist FullDay Option gesetzt UND Zeitdiff >= 1 Tag ? } - $fdopt = 1 if($hash->{LASTCMD} =~ /reduceLog/x); # reduceLog -> FullDay Option damit der ganze Tag berücksichtigt wird + $fdopt = 1 if($hash->{LASTCMD} =~ /reduceLog.*=day/x); # reduceLog -> FullDay Option damit der ganze Tag berücksichtigt wird Log3($name, 4, "DbRep $name - FullDay option: $fdopt"); @@ -10832,13 +11311,11 @@ sub DbRep_corrRelTime { my ($dsec,$dmin,$dhour,$dmday,$dmon,$dyear,$dwday,$dyday,$disdst,$fyear,$cyear); (undef,undef,undef,undef,undef,$cyear,undef,undef,$isdst) = localtime(time); # aktuelles Jahr, Sommer/Winterzeit - if($tdtn) { - # timeDiffToNow + if($tdtn) { # timeDiffToNow ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,undef) = localtime(time); # Istzeit ($dsec,$dmin,$dhour,$dmday,$dmon,$dyear,$dwday,$dyday,$disdst) = localtime(time-$tim); # Istzeit abzgl. Differenzzeit = Selektionsbeginnzeit } - else { - # timeOlderThan + else { # timeOlderThan ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$disdst) = localtime(time-$tim); # Berechnung Selektionsendezeit my $mints = $hash->{HELPER}{MINTS}?$hash->{HELPER}{MINTS}:"1970-01-01 01:00:00"; # Selektionsstartzeit my ($yyyy1, $mm1, $dd1, $hh1, $min1, $sec1) = ($mints =~ /(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)/); @@ -12466,6 +12943,21 @@ sub DbRep_numval { return $val; } +################################################################ +# Prüfung auf numerischen Wert (vorzeichenbehaftet) +################################################################ +sub DbRep_IsNumeric { + my $val = shift // q{empty}; + + my $ret = 0; + + if($val =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/xs) { + $ret = 1; + } + +return $ret; +} + ################################################################ # entfernt führende Mullen einer Zahl ################################################################ @@ -14118,23 +14610,37 @@ return; -
  • reduceLog [<no>[:<nn>]] [average[=day]] [EXCLUDE=device1:reading1,device2:reading2,...] [INCLUDE=device:reading]
    +
  • reduceLog [<no>[:<nn>]] [mode] [EXCLUDE=device1:reading1,device2:reading2,...] [INCLUDE=device:reading]
    Reduces historical data sets.

    - Method without option specification

    + Operation without specifying command line operators

    - The data within the time limits defined by the time.* attributes will be - reduced to one entry (the first) per hour per device & reading. + The data is cleaned within the time limits defined by the time.*-attributes. At least one of the time.* attributes must be set (see table below). - The FullDay option (full days are always selected) is used implicitly. - The respective missing time delimitation is calculated by the module in this case. + The respective missing time accrual is determined by the module in this case.
    + The working mode is determined by the optional specification of mode:

    - By optionally specifying average, not only the database will be cleaned up, but - all numeric values of an hour are reduced to a single average value. - With the option average=day, all numeric values of a day are reduced to a single - average value (implies 'average'). -

    + +
    + With the attributes device and reading the data records to be considered can be included or be excluded. Both restrictions reduce the selected data and reduce the @@ -14146,14 +14652,15 @@ return; @@ -14190,14 +14697,14 @@ return;
    - Method with option specification

    + Operation with specification of command line operators

    - Records older than <no> days and (optionally) newer than - <nn> days are considered. - Specifying average not only cleans up the database, but also - all numerical values of an hour are reduced to a single mean value. - With the option average=day all numerical values of a day are reduced to a single - Average value reduced (implies 'average').

    + Es werden Datensätze berücksichtigt die älter sind als <no> Tage und (optional) neuer sind als + <nn> Tage. + Records are considered that are older than <no> days and (optionally) newer than + <nn> days. + The working mode is determined by the optional specification of mode as described above. +

    The additions "EXCLUDE" or "INCLUDE" can be added to exclude or include device/reading combinations in reduceLog and override the "device" and "reading" attributes, which are ignored in this case.
    @@ -14214,10 +14721,10 @@ return;
    Note:
    - Although the function itself is designed non-blocking, the assigned DbLog device should - operated in asynchronous mode to avoid blocking of FHEMWEB (table lock).
    - It is also strongly recommended to use the standard INDEX 'Search_Idx' in the table 'history'. - to put on !
    + Although the function itself is designed non-blocking, the assigned DbLog device should be operated in + asynchronous mode to avoid blocking FHEMWEB (table lock).
    + Furthermore it is strongly recommended to create the standard INDEX 'Search_Idx' in the table + 'history' !
    The processing of this command may take an extremely long time (without INDEX).


    • @@ -16936,22 +17443,36 @@ return; -
  • reduceLog [<no>[:<nn>]] [average[=day]] [EXCLUDE=device1:reading1,device2:reading2,...] [INCLUDE=device:reading]
    +
  • reduceLog [<no>[:<nn>]] [mode] [EXCLUDE=device1:reading1,device2:reading2,...] [INCLUDE=device:reading]
    Reduziert historische Datensätze.

    - Arbeitsweise ohne Optionsangabe

    + Arbeitsweise ohne Angabe von Befehlszeilenoperatoren

    - Es werden die Daten innerhalb der durch die time.*-Attribute bestimmten - Zeitgrenzen auf einen Eintrag (den ersten) pro Stunde je Device & Reading reduziert. - Es muss mindestens eines der time.*-Attribute gesetzt sein (siehe Tabelle unten). - Die FullDay-Option (es werden immer volle Tage selektiert) wird impliziert verwendet. - Die jeweils fehlende Zeitabgrenzung wird in diesem Fall durch das Modul errechnet. + Es werden die Daten innerhalb der durch die time.*-Attribute bestimmten Zeitgrenzen bereinigt. + Es muss mindestens eines der time.*-Attribute gesetzt sein (siehe Tabelle unten). + Die jeweils fehlende Zeitabgrenzung wird in diesem Fall durch das Modul ermittelt.
    + Der Arbeitsmodus wird durch die optionale Angabe von mode bestimmt:

    - Durch die optionale Angabe von average wird nicht nur die Datenbank bereinigt, sondern - alle numerischen Werte einer Stunde werden auf einen einzigen Mittelwert reduziert. - Mit der Option average=day werden alle numerischen Werte eines Tages auf einen einzigen - Mittelwert reduziert (impliziert 'average').

    + +
    Mit den Attributen device und reading können die zu berücksichtigenden Datensätze eingeschlossen bzw. ausgeschlossen werden. Beide Eingrenzungen reduzieren die selektierten Daten und verringern den @@ -16963,14 +17484,15 @@ return; @@ -17009,14 +17531,12 @@ return;
    - Arbeitsweise mit Optionsangabe

    + Arbeitsweise mit Angabe von Befehlszeilenoperatoren

    Es werden Datensätze berücksichtigt die älter sind als <no> Tage und (optional) neuer sind als <nn> Tage. - Durch die Angabe von average wird nicht nur die Datenbank bereinigt, sondern - alle numerischen Werte einer Stunde werden auf einen einzigen Mittelwert reduziert. - Mit der Option average=day werden alle numerischen Werte eines Tages auf einen einzigen - Mittelwert reduziert (impliziert 'average').

    + Der Arbeitsmodus wird durch die optionale Angabe von mode wie oben beschrieben bestimmt. +

    Die Zusätze "EXCLUDE" bzw. "INCLUDE" können ergänzt werden um device/reading Kombinationen in reduceLog auszuschließen bzw. einzuschließen und überschreiben die Einstellung der Attribute "device" und "reading", die in diesem Fall