From 6b1a9a9b9954ab58208244a608289bad9c882b4f Mon Sep 17 00:00:00 2001 From: nasseeder1 <> Date: Fri, 30 Sep 2016 12:58:21 +0000 Subject: [PATCH] 93_DbRep: include day before and next in calculation if Timestamp is exactly 'YYYY-MM-DD 00:00:00' git-svn-id: https://svn.fhem.de/fhem/trunk@12224 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 2 + fhem/FHEM/93_DbRep.pm | 190 +++++++++++++++++++++++------------------- 2 files changed, 106 insertions(+), 86 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index 0c7261cfc..a996dd6db 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,7 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - bugfix: 93_DbRep: include day before and next in calculation if Timestamp + is exactly 'YYYY-MM-DD 00:00:00' - feature: 93_DbRep: Internal "LASTCMD" added, new reading background_processing_time, diffValue calculation moved to backgrd to reduce load diff --git a/fhem/FHEM/93_DbRep.pm b/fhem/FHEM/93_DbRep.pm index 04ca20796..f63d284a3 100644 --- a/fhem/FHEM/93_DbRep.pm +++ b/fhem/FHEM/93_DbRep.pm @@ -36,7 +36,8 @@ # ########################################################################################################### # Versions History: -# +# 3.11.1 30.09.2016 bugfix include first and next day in calculation if Timestamp is exactly 'YYYY-MM-DD 00:00:00" +# 3.11 29.09.2016 maxValue calculation moved to background to reduce FHEM-load # 3.10.1 28.09.2016 sub impFile -> changed $dbh->{AutoCommit} = 0 to $dbh->begin_work # 3.10 27.09.2016 diffValue calculation moved to background to reduce FHEM-load, # new reading background_processing_time @@ -820,7 +821,7 @@ sub averval_DoParse($) { $sql .= "AND " if($device && $reading); $sql .= "READING = '$reading' " if($reading); $sql .= "AND " if((AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME}, "timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)) && ($device || $reading)); - $sql .= "TIMESTAMP BETWEEN '$runtime_string_first' AND '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); + $sql .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); $sql .= ";"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql"); @@ -983,7 +984,7 @@ sub count_DoParse($) { $sql .= "AND " if($device && $reading); $sql .= "READING = '$reading' " if($reading); $sql .= "AND " if((AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME}, "timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)) && ($device || $reading)); - $sql .= "TIMESTAMP BETWEEN '$runtime_string_first' AND '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); + $sql .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); $sql .= ";"; Log3($name, 4, "DbRep $name - SQL to execute: $sql"); @@ -1144,13 +1145,13 @@ sub maxval_DoParse($) { my $sql = "SELECT VALUE,TIMESTAMP FROM `history` where "; $sql .= "`DEVICE` = '$device' AND " if($device); $sql .= "`READING` = '$reading' AND " if($reading); - $sql .= "TIMESTAMP BETWEEN ? AND ? ORDER BY TIMESTAMP ;"; + $sql .= "TIMESTAMP >= ? AND TIMESTAMP < ? ORDER BY TIMESTAMP ;"; # SQL zusammenstellen für Logausgabe my $sql1 = "SELECT VALUE,TIMESTAMP FROM `history` where "; $sql1 .= "`DEVICE` = '$device' AND " if($device); $sql1 .= "`READING` = '$reading' AND " if($reading); - $sql1 .= "TIMESTAMP BETWEEN '$runtime_string_first' AND '$runtime_string_next' ORDER BY TIMESTAMP;"; + $sql1 .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' ORDER BY TIMESTAMP;"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql1"); @@ -1187,11 +1188,66 @@ sub maxval_DoParse($) { $dbh->disconnect; - my $rowlist = join('|', @row_array); - Log3 ($name, 5, "DbRep $name -> row_array: @row_array"); + Log3 ($name, 5, "DbRep $name -> raw data of row_array result:\n @row_array"); + + #---------- Berechnung Ergebnishash maxValue ------------------------ + my $i = 1; + my %rh = (); + my $lastruntimestring; + my $row_max_time; + my $max_value = 0; + + foreach my $row (@row_array) { + my @a = split("[ \t][ \t]*", $row); + my $runtime_string = decode_base64($a[0]); + $lastruntimestring = $runtime_string if ($i == 1); + my $value = $a[1]; + + $a[3] =~ s/:/-/g if($a[3]); # substituieren unsopported characters -> siehe fhem.pl + my $timestamp = $a[3]?$a[2]."_".$a[3]:$a[2]; + + # Leerzeichen am Ende $timestamp entfernen + $timestamp =~ s/\s+$//g; + + # Test auf $value = "numeric" + if (!looks_like_number($value)) { + $a[3] =~ s/\s+$//g; + Log3 ($name, 2, "DbRep $name - ERROR - value isn't numeric in diffValue function. Faulty dataset was \nTIMESTAMP: $timestamp, DEVICE: $device, READING: $reading, VALUE: $value."); + $err = encode_base64("Value isn't numeric. Faulty dataset was - TIMESTAMP: $timestamp, VALUE: $value", ""); + Log3 ($name, 4, "DbRep $name -> BlockingCall maxval_DoParse finished"); + return "$name|''|$device|$reading|''|$err"; + } + + Log3 ($name, 5, "DbRep $name - Runtimestring: $runtime_string, DEVICE: $device, READING: $reading, TIMESTAMP: $timestamp, VALUE: $value"); + + if ($runtime_string eq $lastruntimestring) { + if ($value >= $max_value) { + $max_value = $value; + $row_max_time = $timestamp; + $rh{$runtime_string} = $runtime_string."|".$max_value."|".$row_max_time; + } + } else { + # neuer Zeitabschnitt beginnt, ersten Value-Wert erfassen + $lastruntimestring = $runtime_string; + $max_value = 0; + if ($value >= $max_value) { + $max_value = $value; + $row_max_time = $timestamp; + $rh{$runtime_string} = $runtime_string."|".$max_value."|".$row_max_time; + } + } + $i++; + } + #--------------------------------------------------------------------------------------------- - # Daten müssen als Einzeiler zurückgegeben werden - $rowlist = encode_base64($rowlist,""); + Log3 ($name, 5, "DbRep $name - result of maxValue calculation before encoding:"); + foreach my $key (sort(keys(%rh))) { + Log3 ($name, 5, "runtimestring Key: $key, value: ".$rh{$key}); + } + + # Ergebnishash als Einzeiler zurückgeben + my $rows = join('§', %rh); + my $rowlist = encode_base64($rows,""); Log3 ($name, 4, "DbRep $name -> BlockingCall maxval_DoParse finished"); @@ -1222,7 +1278,7 @@ sub maxval_ParseDone($) { Log3 ($name, 4, "DbRep $name -> Start BlockingCall maxval_ParseDone"); - if ($err) { + if ($err) { readingsSingleUpdate($hash, "errortext", $err, 1); readingsSingleUpdate($hash, "state", "error", 1); delete($hash->{HELPER}{RUNNING_PID}); @@ -1230,57 +1286,11 @@ sub maxval_ParseDone($) { return; } - my @row_array = split("\\|", $rowlist); - - Log3 ($name, 5, "DbRep $name - row_array decoded: @row_array"); - - my $i = 1; - my %rh = (); - my $lastruntimestring; - my $row_max_time; - my $max_value = 0; - - foreach my $row (@row_array) { - my @a = split("[ \t][ \t]*", $row); - my $runtime_string = decode_base64($a[0]); - $lastruntimestring = $runtime_string if ($i == 1); - my $value = $a[1]; - - # Test auf $value = "numeric" - if (!looks_like_number($value)) { - readingsSingleUpdate($hash, "state", "error", 1); - delete($hash->{HELPER}{RUNNING_PID}); - $a[3] =~ s/\s+$//g; - Log3 ($name, 2, "DbRep $name - ERROR - value isn't numeric in maxValue function. Faulty dataset was \nTIMESTAMP: $a[2] $a[3], DEVICE: $device, READING: $reading, VALUE: $value. \nLeaving ..."); - Log3 ($name, 4, "DbRep $name -> BlockingCall maxval_ParseDone finished"); - return; - } - - $a[3] =~ s/:/-/g if($a[3]); # substituieren unsopported characters -> siehe fhem.pl - my $timestamp = $a[3]?$a[2]."_".$a[3]:$a[2]; - - # Leerzeichen am Ende $timestamp entfernen - $timestamp =~ s/\s+$//g; - - Log3 ($name, 4, "DbRep $name - Runtimestring: $runtime_string, DEVICE: $device, READING: $reading, TIMESTAMP: $timestamp, VALUE: $value"); - - if ($runtime_string eq $lastruntimestring) { - if ($value >= $max_value) { - $max_value = $value; - $row_max_time = $timestamp; - $rh{$runtime_string} = $runtime_string."|".$max_value."|".$row_max_time; - } - } else { - # neuer Zeitabschnitt beginnt, ersten Value-Wert erfassen - $lastruntimestring = $runtime_string; - $max_value = 0; - if ($value >= $max_value) { - $max_value = $value; - $row_max_time = $timestamp; - $rh{$runtime_string} = $runtime_string."|".$max_value."|".$row_max_time; - } - } - $i++; + my %rh = split("§", $rowlist); + + Log3 ($name, 5, "DbRep $name - result of maxValue calculation after decoding:"); + foreach my $key (sort(keys(%rh))) { + Log3 ($name, 5, "DbRep $name - runtimestring Key: $key, value: ".$rh{$key}); } # Readingaufbereitung @@ -1290,7 +1300,6 @@ sub maxval_ParseDone($) { no warnings 'uninitialized'; foreach my $key (sort(keys(%rh))) { - Log3 ($name, 5, "DbRep $name - runtimestring Key: $key, value: ".$rh{$key}); my @k = split("\\|",$rh{$key}); my $rsf = $k[2]."__" if($k[2]); @@ -1303,7 +1312,6 @@ sub maxval_ParseDone($) { } my $rv = $k[1]; readingsBulkUpdate($hash, $reading_runtime_string, $rv?sprintf("%.4f",$rv):"-"); - } readingsBulkUpdate($hash, "background_processing_time", sprintf("%.4f",$brt)) if(AttrVal($name, "showproctime", undef)); @@ -1369,13 +1377,13 @@ sub diffval_DoParse($) { my $sql = "SELECT TIMESTAMP,VALUE FROM `history` where "; $sql .= "`DEVICE` = '$device' AND " if($device); $sql .= "`READING` = '$reading' AND " if($reading); - $sql .= "TIMESTAMP BETWEEN ? AND ? ORDER BY TIMESTAMP ;"; + $sql .= "TIMESTAMP >= ? AND TIMESTAMP < ? ORDER BY TIMESTAMP ;"; # SQL zusammenstellen für Logausgabe my $sql1 = "SELECT TIMESTAMP,VALUE FROM `history` where "; $sql1 .= "`DEVICE` = '$device' AND " if($device); $sql1 .= "`READING` = '$reading' AND " if($reading); - $sql1 .= "TIMESTAMP BETWEEN '$runtime_string_first' AND '$runtime_string_next' ORDER BY TIMESTAMP;"; + $sql1 .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' ORDER BY TIMESTAMP;"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql1"); @@ -1436,6 +1444,7 @@ sub diffval_DoParse($) { # Leerzeichen am Ende $timestamp entfernen $timestamp =~ s/\s+$//g; + # Test auf $value = "numeric" if (!looks_like_number($value)) { $a[3] =~ s/\s+$//g; @@ -1617,7 +1626,7 @@ sub sumval_DoParse($) { $sql .= "AND " if($device && $reading); $sql .= "READING = '$reading' " if($reading); $sql .= "AND " if((AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME}, "timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)) && ($device || $reading)); - $sql .= "TIMESTAMP BETWEEN '$runtime_string_first' AND '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); + $sql .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' " if(AttrVal($hash->{NAME}, "aggregation", "no") ne "no" || $runtime_string_first || AttrVal($hash->{NAME},"timestamp_end",undef) || AttrVal($hash->{NAME},"timeDiffToNow",undef) || AttrVal($hash->{NAME},"timeOlderThan",undef)); $sql .= ";"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql"); @@ -1758,13 +1767,13 @@ sub del_DoParse($) { my $sql = "DELETE FROM history where "; $sql .= "DEVICE = '$device' AND " if($device); $sql .= "READING = '$reading' AND " if($reading); - $sql .= "TIMESTAMP BETWEEN ? AND ?;"; + $sql .= "TIMESTAMP >= ? AND TIMESTAMP < ? ORDER BY TIMESTAMP ;"; # SQL zusammenstellen für Logausgabe my $sql1 = "DELETE FROM history where "; $sql1 .= "DEVICE = '$device' AND " if($device); $sql1 .= "READING = '$reading' AND " if($reading); - $sql1 .= "TIMESTAMP BETWEEN $runtime_string_first AND $runtime_string_next;"; + $sql1 .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next';"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql1"); @@ -2013,13 +2022,13 @@ sub fetchrows_DoParse($) { my $sql = "SELECT DEVICE,READING,TIMESTAMP,VALUE FROM history where "; $sql .= "DEVICE = '$device' AND " if($device); $sql .= "READING = '$reading' AND " if($reading); - $sql .= "TIMESTAMP BETWEEN ? AND ? ORDER BY TIMESTAMP;"; + $sql .= "TIMESTAMP >= ? AND TIMESTAMP < ? ORDER BY TIMESTAMP ;"; # SQL zusammenstellen für Logfileausgabe my $sql1 = "SELECT DEVICE,READING,TIMESTAMP,VALUE FROM history where "; $sql1 .= "DEVICE = '$device' AND " if($device); $sql1 .= "READING = '$reading' AND " if($reading); - $sql1 .= "TIMESTAMP BETWEEN $runtime_string_first AND $runtime_string_next ORDER BY TIMESTAMP;"; + $sql1 .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' ORDER BY TIMESTAMP;"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql1"); @@ -2167,13 +2176,13 @@ sub expfile_DoParse($) { my $sql = "SELECT TIMESTAMP,DEVICE,TYPE,EVENT,READING,VALUE,UNIT FROM history where "; $sql .= "DEVICE = '$device' AND " if($device); $sql .= "READING = '$reading' AND " if($reading); - $sql .= "TIMESTAMP BETWEEN ? AND ? ORDER BY TIMESTAMP;"; + $sql .= "TIMESTAMP >= ? AND TIMESTAMP < ? ORDER BY TIMESTAMP ;"; # SQL zusammenstellen für Logfileausgabe my $sql1 = "SELECT TIMESTAMP,DEVICE,TYPE,EVENT,READING,VALUE,UNIT FROM FROM history where "; $sql1 .= "DEVICE = '$device' AND " if($device); $sql1 .= "READING = '$reading' AND " if($reading); - $sql1 .= "TIMESTAMP BETWEEN $runtime_string_first AND $runtime_string_next ORDER BY TIMESTAMP;"; + $sql1 .= "TIMESTAMP >= '$runtime_string_first' AND TIMESTAMP < '$runtime_string_next' ORDER BY TIMESTAMP;"; Log3 ($name, 4, "DbRep $name - SQL to execute: $sql1"); @@ -2732,7 +2741,8 @@ return ($runtime,$runtime_string,$runtime_string_first,$runtime_string_next,$ll) -
  • sumValue - calculates the amount of readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow". The reading to evaluate must be defined using attribute "reading". Using this function is mostly reasonable if value-differences of readings are written to the database.

    • -
  • maxValue - calculates the maximum value of readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow". The reading to evaluate must be defined using attribute "reading". The evaluation contains the timestamp of the identified max values within the given period.

    • -
  • diffValue - calculates the defference of the readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow". The reading to evaluate must be defined using attribute "reading". This function is mostly reasonable if readingvalues are increasing permanently and don't write value-differences to the database.

    • -
  • delEntries - deletes all database entries or only the database entries specified by attributes Device and/or Reading and the entered time period between "timestamp_begin", "timestamp_end" (if set) or "timeDiffToNow".

    +
  • sumValue - calculates the amount of readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow / timeOlderThan". The reading to evaluate must be defined using attribute "reading". Using this function is mostly reasonable if value-differences of readings are written to the database.

    • +
  • maxValue - calculates the maximum value of readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow / timeOlderThan". The reading to evaluate must be defined using attribute "reading". The evaluation contains the timestamp of the identified max values within the given period.

    • +
  • diffValue - calculates the defference of the readingvalues DB-column "VALUE") between period given by attributes "timestamp_begin", "timestamp_end" or "timeDiffToNow / timeOlderThan". The reading to evaluate must be defined using attribute "reading". + This function is mostly reasonable if readingvalues are increasing permanently and don't write value-differences to the database. + The difference will be generated from the first available dataset (VALUE-Field) to the last available dataset between the specified time linits/aggregation.

    • + +
  • delEntries - deletes all database entries or only the database entries specified by attributes Device and/or Reading and the entered time period between "timestamp_begin", "timestamp_end" (if set) or "timeDiffToNow/timeOlderThan".