FHEM/fhem-mirror
2
0
Fork 0
mirror of https://github.com/fhem/fhem-mirror.git synced 2025-04-08 13:24:56 +00:00
Code Issues Releases Wiki Activity

93_DbRep: contrib v 8.45.0

Browse Source 
git-svn-id: https://svn.fhem.de/fhem/trunk@25296 2b470e98-0d58-463d-a4d8-8e2adae1ed80
This commit is contained in:
nasseeder1 2021-12-04 21:20:24 +00:00
parent f0a32284f7
commit b2b552e55a
1 changed files with 155 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

223
fhem/contrib/DS_Starter/93_DbRep.pm
View File

@ -486,15 +486,18 @@ sub DbRep_Set {
$dir = $dir."/" unless($dir =~ m/\/$/); $dir = $dir."/" unless($dir =~ m/\/$/);
opendir(DIR,$dir); opendir(DIR,$dir);
if ($dbmodel =~ /MYSQL/) { if ($dbmodel =~ /MYSQL/) {
$dbname = $hash->{DATABASE}; $dbname = $hash->{DATABASE};
$sd = $dbname."_.*?(csv|sql)"; $sd = $dbname."_.*?(csv|sql)";
} elsif ($dbmodel =~ /SQLITE/) { }
elsif ($dbmodel =~ /SQLITE/) {
$dbname = $hash->{DATABASE}; $dbname = $hash->{DATABASE};
$dbname = (split /[\/]/, $dbname)[-1]; $dbname = (split /[\/]/, $dbname)[-1];
$dbname = (split /\./, $dbname)[0]; $dbname = (split /\./, $dbname)[0];
$sd = $dbname."_.*?.sqlitebkp"; $sd = $dbname."_.*?.sqlitebkp";
} }
while (my $file = readdir(DIR)) { while (my $file = readdir(DIR)) {
next unless (-f "$dir/$file"); next unless (-f "$dir/$file");
next unless ($file =~ /^$sd/); next unless ($file =~ /^$sd/);
@ -628,11 +631,12 @@ sub DbRep_Set {
if ($opt =~ m/reduceLog/ && $hash->{ROLE} ne "Agent") { if ($opt =~ m/reduceLog/ && $hash->{ROLE} ne "Agent") {
if ($hash->{HELPER}{RUNNING_REDUCELOG} && $hash->{HELPER}{RUNNING_REDUCELOG}{pid} !~ m/DEAD/) { if ($hash->{HELPER}{RUNNING_REDUCELOG} && $hash->{HELPER}{RUNNING_REDUCELOG}{pid} !~ m/DEAD/) {
return "reduceLog already in progress. Please wait for the current process to finish."; return "reduceLog already in progress. Please wait for the current process to finish.";
} else { }
else {
delete $hash->{HELPER}{RUNNING_REDUCELOG}; delete $hash->{HELPER}{RUNNING_REDUCELOG};
my @b = @a; my @b = @a;
shift(@b); shift(@b);
$hash->{LASTCMD} = join(" ",@b); $hash->{LASTCMD} = join(" ",@b);
$hash->{HELPER}{REDUCELOG} = \@a; $hash->{HELPER}{REDUCELOG} = \@a;
Log3 ($name, 3, "DbRep $name - ################################################################"); Log3 ($name, 3, "DbRep $name - ################################################################");
Log3 ($name, 3, "DbRep $name - ### new reduceLog run ###"); Log3 ($name, 3, "DbRep $name - ### new reduceLog run ###");
@ -8981,8 +8985,8 @@ sub DbRep_reduceLog {
my $dbmodel = $dbloghash->{MODEL}; my $dbmodel = $dbloghash->{MODEL};
my $dbpassword = $attr{"sec$dblogname"}{secret}; my $dbpassword = $attr{"sec$dblogname"}{secret};
my @a = @{$hash->{HELPER}{REDUCELOG}}; my @a = @{$hash->{HELPER}{REDUCELOG}};
my $rlpar = join(" ", @a); my $rlpar = join " ", @a;
my $utf8 = defined($hash->{UTF8})?$hash->{UTF8}:0; my $utf8 = defined($hash->{UTF8}) ? $hash->{UTF8} : 0;
my ($ret,$row,$filter,$exclude,$c,$day,$hour,$lastHour,$updDate,$updHour,$average,$processingDay,$lastUpdH); my ($ret,$row,$filter,$exclude,$c,$day,$hour,$lastHour,$updDate,$updHour,$average,$processingDay,$lastUpdH);
my (%hourlyKnown,%averageHash,@excludeRegex,@dayRows,@averageUpd,@averageUpdD); my (%hourlyKnown,%averageHash,@excludeRegex,@dayRows,@averageUpd,@averageUpdD);
@ -8993,26 +8997,31 @@ sub DbRep_reduceLog {
# ausfiltern von optionalen Zeitangaben, z.B. 700:750 -> Rest zur Einhaltung des ursprünglichen Formats nach @a schreiben # ausfiltern von optionalen Zeitangaben, z.B. 700:750 -> Rest zur Einhaltung des ursprünglichen Formats nach @a schreiben
my @b; my @b;
foreach (@a) { for my $w (@a) {
next if($_ =~ /\b(\d+(:\d+)?)\b/); next if($w =~ /\b(\d+(:\d+)?)\b/);
push @b, $_; push @b, $w;
} }
@a = @b; @a = @b;
Log3 ($name, 5, "DbRep $name -> Start DbLog_reduceLog"); Log3 ($name, 5, "DbRep $name -> Start DbLog_reduceLog");
eval {$dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1, AutoInactiveDestroy => 1, mysql_enable_utf8 => $utf8 });}; eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0,
RaiseError => 1,
if ($@) { AutoInactiveDestroy => 1,
$err = encode_base64($@,""); mysql_enable_utf8 => $utf8
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); }
return "$name|''|$err|''"; ); 1;
} }
or do { $err = encode_base64($@,"");
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
return "$name|''|$err|''";
};
if ($a[-1] =~ /^EXCLUDE=(.+:.+)+/i) { if ($a[-1] =~ /^EXCLUDE=(.+:.+)+/i) {
($filter) = $a[-1] =~ /^EXCLUDE=(.+)/i; ($filter) = $a[-1] =~ /^EXCLUDE=(.+)/i;
@excludeRegex = split(',',$filter); @excludeRegex = split(',',$filter);
} elsif ($a[-1] =~ /^INCLUDE=.+:.+$/i) { }
elsif ($a[-1] =~ /^INCLUDE=.+:.+$/i) {
$filter = 1; $filter = 1;
} }
@ -9026,21 +9035,26 @@ sub DbRep_reduceLog {
my $selspec = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM history where "; my $selspec = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM history where ";
my $addon = "ORDER BY TIMESTAMP ASC"; my $addon = "ORDER BY TIMESTAMP ASC";
if($filter) { if($filter) { # Option EX/INCLUDE wurde angegeben
# Option EX/INCLUDE wurde angegeben
$sql = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM history WHERE " $sql = "SELECT TIMESTAMP,DEVICE,'',READING,VALUE FROM history WHERE "
.($a[-1] =~ /^INCLUDE=(.+):(.+)$/i ? "DEVICE like '$1' AND READING like '$2' AND " : '') .($a[-1] =~ /^INCLUDE=(.+):(.+)$/i ? "DEVICE like '$1' AND READING like '$2' AND " : '')
."TIMESTAMP <= '$ots'".($nts?" AND TIMESTAMP >= '$nts' ":" ")."ORDER BY TIMESTAMP ASC"; ."TIMESTAMP <= '$ots'"
} elsif ($IsTimeSet || $IsAggrSet) { .($nts ? " AND TIMESTAMP >= '$nts' " : " ")
."ORDER BY TIMESTAMP ASC";
}
elsif ($IsTimeSet || $IsAggrSet) {
$sql = DbRep_createCommonSql($hash,$selspec,$d,$r,"'$nts'","'$ots'",$addon); $sql = DbRep_createCommonSql($hash,$selspec,$d,$r,"'$nts'","'$ots'",$addon);
} else { }
else {
$sql = DbRep_createCommonSql($hash,$selspec,$d,$r,undef,undef,$addon); $sql = DbRep_createCommonSql($hash,$selspec,$d,$r,undef,undef,$addon);
} }
Log3 ($name, 4, "DbRep $name - SQL execute: $sql"); Log3 ($name, 4, "DbRep $name - SQL execute: $sql");
if (defined($a[2])) { if (defined($a[2])) {
$average = ($a[2] =~ /average=day/i) ? "AVERAGE=DAY" : ($a[2] =~ /average/i) ? "AVERAGE=HOUR" : 0; $average = ($a[2] =~ /average=day/i) ? "AVERAGE=DAY" :
($a[2] =~ /average/i) ? "AVERAGE=HOUR" :
0;
} }
Log3 ($name, 3, "DbRep $name - reduce data older than: $ots, newer than: $nts"); Log3 ($name, 3, "DbRep $name - reduce data older than: $ots, newer than: $nts");
@ -9048,9 +9062,9 @@ sub DbRep_reduceLog {
.(($average) ? "$average" : '') .(($average) ? "$average" : '')
.($filter ? ((($average && $filter) ? ", " : '').(uc((split('=',$a[-1]))[0]).'='.(split('=',$a[-1]))[1])) : .($filter ? ((($average && $filter) ? ", " : '').(uc((split('=',$a[-1]))[0]).'='.(split('=',$a[-1]))[1])) :
((($idanz || $idevswc || $iranz || $irdswc) ? " INCLUDE -> " : ''). ((($idanz || $idevswc || $iranz || $irdswc) ? " INCLUDE -> " : '').
(($idanz || $idevswc)? "Devs: ".($idevs?$idevs:'').($idevswc?$idevswc:'') : '').(($iranz || $irdswc) ? " Readings: ".($ireading?$ireading:'').($irdswc?$irdswc:'') : ''). (($idanz || $idevswc) ? "Devs: ".($idevs ? $idevs:'').($idevswc ? $idevswc : '') : '').(($iranz || $irdswc) ? " Readings: ".($ireading ? $ireading : '').($irdswc ? $irdswc : '') : '').
(($edanz || $edevswc || $eranz || $erdswc) ? " EXCLUDE -> " : ''). (($edanz || $edevswc || $eranz || $erdswc) ? " EXCLUDE -> " : '').
(($edanz || $edevswc)? "Devs: ".($edevs?$edevs:'').($edevswc?$edevswc:'') : '').(($eranz || $erdswc) ? " Readings: ".($ereading?$ereading:'').($erdswc?$erdswc:'') : '')) )); (($edanz || $edevswc) ? "Devs: ".($edevs ? $edevs : '').($edevswc ? $edevswc : '') : '').(($eranz || $erdswc) ? " Readings: ".($ereading ? $ereading:'').($erdswc ? $erdswc : '') : '')) ));
if ($ots) { if ($ots) {
my ($sth_del, $sth_upd, $sth_delD, $sth_updD, $sth_get); my ($sth_del, $sth_upd, $sth_delD, $sth_updD, $sth_get);
@ -9074,54 +9088,73 @@ sub DbRep_reduceLog {
} }
do { do {
$row = $sth_get->fetchrow_arrayref || ['0000-00-00 00:00:00','D','','R','V']; # || execute last-day dummy $row = $sth_get->fetchrow_arrayref || ['0000-00-00 00:00:00','D','','R','V']; # || execute last-day dummy
$ret = 1; $ret = 1;
($day,$hour) = $row->[0] =~ /-(\d{2})\s(\d{2}):/; ($day,$hour) = $row->[0] =~ /-(\d{2})\s(\d{2}):/;
$rowCount++ if($day != 00); $rowCount++ if($day != 00);
if ($day != $currentDay) { if ($day != $currentDay) {
if ($currentDay) { # false on first executed day if ($currentDay) { # false on first executed day
if (scalar @dayRows) { if (scalar @dayRows) {
($lastHour) = $dayRows[-1]->[0] =~ /(.*\d+\s\d{2}):/; ($lastHour) = $dayRows[-1]->[0] =~ /(.*\d+\s\d{2}):/;
$c = 0; $c = 0;
for my $delRow (@dayRows) { for my $delRow (@dayRows) {
$c++ if($day != 00 || $delRow->[0] !~ /$lastHour/); $c++ if($day != 00 || $delRow->[0] !~ /$lastHour/);
} }
if($c) { if($c) {
$deletedCount += $c; $deletedCount += $c;
Log3 ($name, 3, "DbRep $name - reduceLog deleting $c records of day: $processingDay"); Log3 ($name, 3, "DbRep $name - reduceLog deleting $c records of day: $processingDay");
$dbh->{RaiseError} = 1; $dbh->{RaiseError} = 1;
$dbh->{PrintError} = 0; $dbh->{PrintError} = 0;
eval {$dbh->begin_work() if($dbh->{AutoCommit});}; eval {$dbh->begin_work() if($dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
eval { eval {
my $i = 0; my $i = 0;
my $k = 1; my $k = 1;
my $th = ($#dayRows <= 2000)?100:($#dayRows <= 30000)?1000:10000; my $th = ($#dayRows <= 2000) ? 100 :
($#dayRows <= 30000) ? 1000 :
10000;
for my $delRow (@dayRows) { for my $delRow (@dayRows) {
if($day != 00 || $delRow->[0] !~ /$lastHour/) { if($day != 00 || $delRow->[0] !~ /$lastHour/) {
Log3 ($name, 4, "DbRep $name - DELETE FROM history WHERE (DEVICE=$delRow->[1]) AND (READING=$delRow->[3]) AND (TIMESTAMP=$delRow->[0]) AND (VALUE=$delRow->[4])"); Log3 ($name, 4, "DbRep $name - DELETE FROM history WHERE (DEVICE=$delRow->[1]) AND (READING=$delRow->[3]) AND (TIMESTAMP=$delRow->[0]) AND (VALUE=$delRow->[4])");
$sth_del->execute(($delRow->[1], $delRow->[3], $delRow->[0], $delRow->[4])); $sth_del->execute(($delRow->[1], $delRow->[3], $delRow->[0], $delRow->[4]));
$i++; $i++;
if($i == $th) { if($i == $th) {
my $prog = $k * $i; my $prog = $k * $i;
Log3 ($name, 3, "DbRep $name - reduceLog deletion progress of day: $processingDay is: $prog"); Log3 ($name, 3, "DbRep $name - reduceLog deletion progress of day: $processingDay is: $prog");
$i = 0; $i = 0;
$k++; $k++;
} }
} }
} }
}; };
if ($@) { if ($@) {
$err = $@; $err = $@;
Log3 ($name, 2, "DbRep $name - reduceLog ! FAILED ! for day $processingDay: $err"); Log3 ($name, 2, "DbRep $name - reduceLog ! FAILED ! for day $processingDay: $err");
eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; eval {$dbh->rollback() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
$ret = 0; $ret = 0;
} else { }
else {
eval {$dbh->commit() if(!$dbh->{AutoCommit});}; eval {$dbh->commit() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
@ -9136,46 +9169,62 @@ sub DbRep_reduceLog {
if ($ret && defined($a[2]) && $a[2] =~ /average/i) { if ($ret && defined($a[2]) && $a[2] =~ /average/i) {
$dbh->{RaiseError} = 1; $dbh->{RaiseError} = 1;
$dbh->{PrintError} = 0; $dbh->{PrintError} = 0;
eval {$dbh->begin_work() if($dbh->{AutoCommit});}; eval {$dbh->begin_work() if($dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
eval { eval {
push(@averageUpd, {%hourlyKnown}) if($day != 00); push(@averageUpd, {%hourlyKnown}) if($day != 00);
$c = 0; $c = 0;
for my $hourHash (@averageUpd) { # Only count for logging... for my $hourHash (@averageUpd) { # Only count for logging...
for my $hourKey (keys %$hourHash) { for my $hourKey (keys %$hourHash) {
$c++ if ($hourHash->{$hourKey}->[0] && scalar(@{$hourHash->{$hourKey}->[4]}) > 1); $c++ if ($hourHash->{$hourKey}->[0] && scalar(@{$hourHash->{$hourKey}->[4]}) > 1);
} }
} }
$updateCount += $c;
Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating $c records of day: $processingDay") if($c); # else only push to @averageUpdD
my $i = 0; $updateCount += $c;
my $k = 1;
my $th = ($c <= 2000)?100:($c <= 30000)?1000:10000; Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating $c records of day: $processingDay") if($c); # else only push to @averageUpdD
my $i = 0;
my $k = 1;
my $th = ($c <= 2000) ? 100 : ($c <= 30000) ? 1000 : 10000;
for my $hourHash (@averageUpd) { for my $hourHash (@averageUpd) {
for my $hourKey (keys %$hourHash) { for my $hourKey (keys %$hourHash) {
if ($hourHash->{$hourKey}->[0]) { # true if reading is a number if ($hourHash->{$hourKey}->[0]) { # true if reading is a number
($updDate,$updHour) = $hourHash->{$hourKey}->[0] =~ /(.*\d+)\s(\d{2}):/; ($updDate,$updHour) = $hourHash->{$hourKey}->[0] =~ /(.*\d+)\s(\d{2}):/;
if (scalar(@{$hourHash->{$hourKey}->[4]}) > 1) { # true if reading has multiple records this hour
if (scalar(@{$hourHash->{$hourKey}->[4]}) > 1) { # true if reading has multiple records this hour
for (@{$hourHash->{$hourKey}->[4]}) { $sum += $_; } for (@{$hourHash->{$hourKey}->[4]}) { $sum += $_; }
$average = sprintf('%.3f', $sum/scalar(@{$hourHash->{$hourKey}->[4]}) ); $average = sprintf('%.3f', $sum/scalar(@{$hourHash->{$hourKey}->[4]}) );
$sum = 0; $sum = 0;
Log3 ($name, 4, "DbRep $name - UPDATE history SET TIMESTAMP=$updDate $updHour:30:00, EVENT='rl_av_h', VALUE=$average WHERE DEVICE=$hourHash->{$hourKey}->[1] AND READING=$hourHash->{$hourKey}->[3] AND TIMESTAMP=$hourHash->{$hourKey}->[0] AND VALUE=$hourHash->{$hourKey}->[4]->[0]"); Log3 ($name, 4, "DbRep $name - UPDATE history SET TIMESTAMP=$updDate $updHour:30:00, EVENT='rl_av_h', VALUE=$average WHERE DEVICE=$hourHash->{$hourKey}->[1] AND READING=$hourHash->{$hourKey}->[3] AND TIMESTAMP=$hourHash->{$hourKey}->[0] AND VALUE=$hourHash->{$hourKey}->[4]->[0]");
$sth_upd->execute(("$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[4]->[0])); $sth_upd->execute(("$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[4]->[0]));
$i++; $i++;
if($i == $th) { if($i == $th) {
my $prog = $k * $i; my $prog = $k * $i;
Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating progress of day: $processingDay is: $prog"); Log3 ($name, 3, "DbRep $name - reduceLog (hourly-average) updating progress of day: $processingDay is: $prog");
$i = 0; $i = 0;
$k++; $k++;
} }
push(@averageUpdD, ["$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]) if (defined($a[3]) && $a[3] =~ /average=day/i);
} else { if (defined($a[3]) && $a[3] =~ /average=day/i) {
push(@averageUpdD, [$hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[2], $hourHash->{$hourKey}->[4]->[0], $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]) if (defined($a[3]) && $a[3] =~ /average=day/i); push(@averageUpdD, ["$updDate $updHour:30:00", 'rl_av_h', $average, $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]);
}
}
else {
if (defined($a[3]) && $a[3] =~ /average=day/i) {
push(@averageUpdD, [$hourHash->{$hourKey}->[0], $hourHash->{$hourKey}->[2], $hourHash->{$hourKey}->[4]->[0], $hourHash->{$hourKey}->[1], $hourHash->{$hourKey}->[3], $updDate]);
}
} }
} }
} }
@ -9183,30 +9232,36 @@ sub DbRep_reduceLog {
}; };
if ($@) { if ($@) {
$err = $@; $err = $@;
Log3 ($name, 2, "DbRep $name - reduceLog average=hour ! FAILED ! for day $processingDay: $err"); Log3 ($name, 2, "DbRep $name - reduceLog average=hour ! FAILED ! for day $processingDay: $err");
eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; eval {$dbh->rollback() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
@averageUpdD = (); @averageUpdD = ();
} else { }
else {
eval {$dbh->commit() if(!$dbh->{AutoCommit});}; eval {$dbh->commit() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
} }
$dbh->{RaiseError} = 0; $dbh->{RaiseError} = 0;
$dbh->{PrintError} = 1; $dbh->{PrintError} = 1;
@averageUpd = (); @averageUpd = ();
} }
if (defined($a[2]) && $a[2] =~ /average=day/i && scalar(@averageUpdD) && $day != 00) { if (defined($a[2]) && $a[2] =~ /average=day/i && scalar(@averageUpdD) && $day != 00) {
$dbh->{RaiseError} = 1; $dbh->{RaiseError} = 1;
$dbh->{PrintError} = 0; $dbh->{PrintError} = 0;
eval {$dbh->begin_work() if($dbh->{AutoCommit});}; eval {$dbh->begin_work() if($dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
eval { eval {
for (@averageUpdD) { for (@averageUpdD) {
push(@{$averageHash{$_->[3].$_->[4]}->{tedr}}, [$_->[0], $_->[1], $_->[3], $_->[4]]); push(@{$averageHash{$_->[3].$_->[4]}->{tedr}}, [$_->[0], $_->[1], $_->[3], $_->[4]]);
@ -9216,25 +9271,35 @@ sub DbRep_reduceLog {
$c = 0; $c = 0;
for (keys %averageHash) { for (keys %averageHash) {
if(scalar @{$averageHash{$_}->{tedr}} == 1) { if(scalar @{$averageHash{$_}->{tedr}} == 1) {
delete $averageHash{$_}; delete $averageHash{$_};
} else { }
else {
$c += (scalar(@{$averageHash{$_}->{tedr}}) - 1); $c += (scalar(@{$averageHash{$_}->{tedr}}) - 1);
} }
} }
$deletedCount += $c; $deletedCount += $c;
$updateCount += keys(%averageHash); $updateCount += keys(%averageHash);
my ($id,$iu) = 0; my ($id,$iu) = 0;
my ($kd,$ku) = 1; my ($kd,$ku) = 1;
my $thd = ($c <= 2000)?100:($c <= 30000)?1000:10000; my $thd = ($c <= 2000) ? 100 :
my $thu = ((keys %averageHash) <= 2000)?100:((keys %averageHash) <= 30000)?1000:10000; ($c <= 30000) ? 1000 :
10000;
my $thu = ((keys %averageHash) <= 2000) ? 100 :
((keys %averageHash) <= 30000) ? 1000 :
10000;
Log3 ($name, 3, "DbRep $name - reduceLog (daily-average) updating ".(keys %averageHash).", deleting $c records of day: $processingDay") if(keys %averageHash); Log3 ($name, 3, "DbRep $name - reduceLog (daily-average) updating ".(keys %averageHash).", deleting $c records of day: $processingDay") if(keys %averageHash);
for my $reading (keys %averageHash) { for my $reading (keys %averageHash) {
$average = sprintf('%.3f', $averageHash{$reading}->{sum}/scalar(@{$averageHash{$reading}->{tedr}})); $average = sprintf('%.3f', $averageHash{$reading}->{sum}/scalar(@{$averageHash{$reading}->{tedr}}));
$lastUpdH = pop @{$averageHash{$reading}->{tedr}}; $lastUpdH = pop @{$averageHash{$reading}->{tedr}};
for (@{$averageHash{$reading}->{tedr}}) { for (@{$averageHash{$reading}->{tedr}}) {
Log3 ($name, 5, "DbRep $name - DELETE FROM history WHERE DEVICE='$_->[2]' AND READING='$_->[3]' AND TIMESTAMP='$_->[0]'"); Log3 ($name, 5, "DbRep $name - DELETE FROM history WHERE DEVICE='$_->[2]' AND READING='$_->[3]' AND TIMESTAMP='$_->[0]'");
$sth_delD->execute(($_->[2], $_->[3], $_->[0])); $sth_delD->execute(($_->[2], $_->[3], $_->[0]));
$id++; $id++;
@ -9245,7 +9310,9 @@ sub DbRep_reduceLog {
$kd++; $kd++;
} }
} }
Log3 ($name, 4, "DbRep $name - UPDATE history SET TIMESTAMP=$averageHash{$reading}->{date} 12:00:00, EVENT='rl_av_d', VALUE=$average WHERE (DEVICE=$lastUpdH->[2]) AND (READING=$lastUpdH->[3]) AND (TIMESTAMP=$lastUpdH->[0])"); Log3 ($name, 4, "DbRep $name - UPDATE history SET TIMESTAMP=$averageHash{$reading}->{date} 12:00:00, EVENT='rl_av_d', VALUE=$average WHERE (DEVICE=$lastUpdH->[2]) AND (READING=$lastUpdH->[3]) AND (TIMESTAMP=$lastUpdH->[0])");
$sth_updD->execute(($averageHash{$reading}->{date}." 12:00:00", 'rl_av_d', $average, $lastUpdH->[2], $lastUpdH->[3], $lastUpdH->[0])); $sth_updD->execute(($averageHash{$reading}->{date}." 12:00:00", 'rl_av_d', $average, $lastUpdH->[2], $lastUpdH->[3], $lastUpdH->[0]));
$iu++; $iu++;
@ -9257,52 +9324,62 @@ sub DbRep_reduceLog {
} }
} }
}; };
if ($@) { if ($@) {
Log3 ($name, 3, "DbRep $name - reduceLog average=day ! FAILED ! for day $processingDay"); Log3 ($name, 3, "DbRep $name - reduceLog average=day ! FAILED ! for day $processingDay");
eval {$dbh->rollback() if(!$dbh->{AutoCommit});}; eval {$dbh->rollback() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
} else { }
else {
eval {$dbh->commit() if(!$dbh->{AutoCommit});}; eval {$dbh->commit() if(!$dbh->{AutoCommit});};
if ($@) { if ($@) {
Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@"); Log3 ($name, 2, "DbRep $name - DbRep_reduceLog - $@");
} }
} }
$dbh->{RaiseError} = 0; $dbh->{RaiseError} = 0;
$dbh->{PrintError} = 1; $dbh->{PrintError} = 1;
} }
%averageHash = (); %averageHash = ();
%hourlyKnown = (); %hourlyKnown = ();
@averageUpd = (); @averageUpd = ();
@averageUpdD = (); @averageUpdD = ();
$currentHour = 99; $currentHour = 99;
} }
$currentDay = $day; $currentDay = $day;
} }
if ($hour != $currentHour) { # forget records from last hour, but remember these for average if ($hour != $currentHour) { # forget records from last hour, but remember these for average
if (defined($a[2]) && $a[2] =~ /average/i && keys(%hourlyKnown)) { if (defined($a[2]) && $a[2] =~ /average/i && keys(%hourlyKnown)) {
push(@averageUpd, {%hourlyKnown}); push(@averageUpd, {%hourlyKnown});
} }
%hourlyKnown = (); %hourlyKnown = ();
$currentHour = $hour; $currentHour = $hour;
} }
if (defined $hourlyKnown{$row->[1].$row->[3]}) { # remember first readings for device per h, other can be deleted
if (defined $hourlyKnown{$row->[1].$row->[3]}) { # remember first readings for device per h, other can be deleted
push(@dayRows, [@$row]); push(@dayRows, [@$row]);
if (defined($a[2]) && $a[2] =~ /average/i && defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/ && $hourlyKnown{$row->[1].$row->[3]}->[0]) { if (defined($a[2]) && $a[2] =~ /average/i && defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/ && $hourlyKnown{$row->[1].$row->[3]}->[0]) {
if ($hourlyKnown{$row->[1].$row->[3]}->[0]) { if ($hourlyKnown{$row->[1].$row->[3]}->[0]) {
push(@{$hourlyKnown{$row->[1].$row->[3]}->[4]}, $row->[4]); push(@{$hourlyKnown{$row->[1].$row->[3]}->[4]}, $row->[4]);
} }
} }
} else { }
else {
$exclude = 0; $exclude = 0;
for (@excludeRegex) { for (@excludeRegex) {
$exclude = 1 if("$row->[1]:$row->[3]" =~ /^$_$/); $exclude = 1 if("$row->[1]:$row->[3]" =~ /^$_$/);
} }
if ($exclude) { if ($exclude) {
$excludeCount++ if($day != 00); $excludeCount++ if($day != 00);
} else { }
else {
$hourlyKnown{$row->[1].$row->[3]} = (defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/) ? [$row->[0],$row->[1],$row->[2],$row->[3],[$row->[4]]] : [0]; $hourlyKnown{$row->[1].$row->[3]} = (defined($row->[4]) && $row->[4] =~ /^-?(?:\d+(?:\.\d*)?|\.\d+)$/) ? [$row->[0],$row->[1],$row->[2],$row->[3],[$row->[4]]] : [0];
} }
} }
@ -9311,21 +9388,27 @@ sub DbRep_reduceLog {
} while( $day != 00 ); } while( $day != 00 );
$brt = time() - $startTime; $brt = time() - $startTime;
my $result = "Rows processed: $rowCount, deleted: $deletedCount" my $result = "Rows processed: $rowCount, deleted: $deletedCount"
.((defined($a[2]) && $a[2] =~ /average/i)? ", updated: $updateCount" : '') .((defined($a[2]) && $a[2] =~ /average/i) ? ", updated: $updateCount" : '')
.(($excludeCount)? ", excluded: $excludeCount" : ''); .(($excludeCount) ? ", excluded: $excludeCount" : '');
Log3 ($name, 3, "DbRep $name - reduceLog finished. $result"); Log3 ($name, 3, "DbRep $name - reduceLog finished. $result");
$ret = $result; $ret = $result;
$ret = "reduceLog finished. $result"; $ret = "reduceLog finished. $result";
} else { }
$err = "reduceLog needs at least one of attributes \"timeOlderThan\", \"timeDiffToNow\", \"timestamp_begin\" or \"timestamp_end\" to be set"; else {
$err = qq{reduceLog needs at least one of attributes "timeOlderThan", "timeDiffToNow", "timestamp_begin" or "timestamp_end" to be set};
Log3 ($name, 2, "DbRep $name - ERROR - $err"); Log3 ($name, 2, "DbRep $name - ERROR - $err");
$err = encode_base64($err,""); $err = encode_base64($err,"");
return "$name|''|$err|''"; return "$name|''|$err|''";
} }
$dbh->disconnect(); $dbh->disconnect();
$ret = encode_base64($ret,""); $ret = encode_base64($ret,"");
Log3 ($name, 5, "DbRep $name -> DbRep_reduceLogNbl finished"); Log3 ($name, 5, "DbRep $name -> DbRep_reduceLogNbl finished");
return "$name|$ret|0|$brt"; return "$name|$ret|0|$brt";
@ -14767,7 +14850,7 @@ return;
<ul> <ul>
<b>Example:</b> <br> <b>Example:</b> <br>
attr <device> userExitFn UserFunction .*:.* <br> attr <device> userExitFn UserFunction Meter:Energy.* <br>
# "UserFunction" is the subroutine in 99_myUtils.pm. # "UserFunction" is the subroutine in 99_myUtils.pm.
</ul> </ul>
<br> <br>
@ -14787,7 +14870,7 @@ return;
<li>$VALUE - the value of the reading </li> <li>$VALUE - the value of the reading </li>
</ul> </ul>
<br><br> <br>
<ul> <ul>
<b>Example:</b> <br> <b>Example:</b> <br>
@ -17465,14 +17548,14 @@ return;
} }
</pre> </pre>
Im im Attribut wird die Subroutine und optional ein Reading:Value Regex Im Attribut wird die Subroutine und optional ein Reading:Value Regex
als Argument angegeben werden. Ohne diese Angabe werden alle Wertekombinationen als "wahr" als Argument angegeben. Ohne diese Angabe werden alle Wertekombinationen als "wahr"
gewertet und an die Subroutine übergeben (entspricht .*:.*). gewertet und an die Subroutine übergeben (entspricht .*:.*).
<br><br> <br><br>
<ul> <ul>
<b>Beispiel:</b> <br> <b>Beispiel:</b> <br>
attr <device> userExitFn UserFunction .*:.* <br> attr <device> userExitFn UserFunction Meter:Energy.* <br>
# "UserFunction" ist die Subroutine in 99_myUtils.pm. # "UserFunction" ist die Subroutine in 99_myUtils.pm.
</ul> </ul>
<br> <br>
@ -17492,7 +17575,7 @@ return;
<li>$VALUE - der Wert des Readings </li> <li>$VALUE - der Wert des Readings </li>
</ul> </ul>
<br><br> <br>
<ul> <ul>
<b>Beispiel:</b> <br> <b>Beispiel:</b> <br>