From 1fc9bacbab692228094919fa66ec78f1d41dd676 Mon Sep 17 00:00:00 2001 From: nasseeder1 Date: Sat, 25 Jan 2020 19:45:16 +0000 Subject: [PATCH] 93_DbRep: adjust behavior of writeToDB - write value at every begin and also at every end of period, fix Warning when Agent has detected a renamed device, remove adminCredentials from set of DbRep device type "Agent", delDoublets now are working for PostgreSQL, calculation of number_fetched_rows corrected, count of inserted rows into standby database corrected, avoid Duplicate entry errors in writeToDB git-svn-id: https://svn.fhem.de/fhem/trunk@21050 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 8 + fhem/FHEM/93_DbRep.pm | 285 +++++++++++++++------------- fhem/contrib/DS_Starter/93_DbRep.pm | 59 ++---- 3 files changed, 181 insertions(+), 171 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index 593d2c045..736036d87 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,5 +1,13 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - change: 93_DbRep: adjust behavior of writeToDB - write value at every begin + and also at every end of period, fix Warning when Agent + has detected a renamed device, remove adminCredentials + from set of DbRep device type "Agent", delDoublets now + are working for PostgreSQL, calculation of + number_fetched_rows corrected, count of inserted rows + into standby database corrected, avoid Duplicate entry + errors in writeToDB - change: 98_todoist: user information in FHEMWEB widget - change: 72_TA_CMI_JSON: added new API versions - bugfix: 88_Timer: fix list "Probably associated with" Forum: #1015549 diff --git a/fhem/FHEM/93_DbRep.pm b/fhem/FHEM/93_DbRep.pm index 58306fc75..390a3551d 100644 --- a/fhem/FHEM/93_DbRep.pm +++ b/fhem/FHEM/93_DbRep.pm @@ -3,7 +3,7 @@ ########################################################################################################## # 93_DbRep.pm # -# (c) 2016-2019 by Heiko Maaz +# (c) 2016-2020 by Heiko Maaz # e-mail: Heiko dot Maaz at t-online dot de # # This Module can be used to select and report content of databases written by 93_DbLog module @@ -58,6 +58,13 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # Version History intern our %DbRep_vNotesIntern = ( + "8.30.8" => "25.01.2020 adjust SQL-Statements in OutputWriteToDB to avoid Duplicate entry errors and other fixes ", + "8.30.7" => "24.01.2020 corrected count of inserted rows into standby database (DbRep_WriteToDB) ", + "8.30.6" => "23.01.2020 delDoublets now are working also for PostgreSQL, calculation of number_fetched_rows corrected ", + "8.30.5" => "23.01.2020 remove adminCredentials from set of device type \"Agent\" ", + "8.30.4" => "22.01.2020 adjust behavior of OutputWriteToDB (averageValue,sumValue) - write value at every begin and also at every end of period ". + "Forum: https://forum.fhem.de/index.php/topic,105787.msg1013920.html#msg1013920 ". + "fix Warning when Agent has detected a renamed device", "8.30.3" => "28.11.2019 countEntries encode \$device, change count_ParseDone for \"countEntriesDetail\" ", "8.30.2" => "24.11.2019 change order of delete(\$hash->{HELPER}{RUNNING_PID}) in *_ParseDone routines, Forum: https://forum.fhem.de/index.php/topic,105591.msg996089.html#msg996089 ", "8.30.1" => "22.11.2019 commandref revised ", @@ -174,6 +181,8 @@ our %DbRep_vNotesIntern = ( # Version History extern: our %DbRep_vNotesExtern = ( + "8.30.4" => "22.01.2020 The behavior of write back values to database is changed for functions averageValue and sumValue. The solution values of that functions now are ". + "written at every begin and also at every end of specified aggregation period. ", "8.30.0" => "14.11.2019 A new command \"set adminCredentials\" and \"get storedCredentials\" ist provided. ". "Use it to store a database priviledged user. This user DbRep can utilize for several operations which are need more (administative) ". "user rights (e.g. index, sqlCmd). ", @@ -544,7 +553,7 @@ sub DbRep_Set($@) { (($hash->{ROLE} ne "Agent")?"fetchrows:history,current ":""). (($hash->{ROLE} ne "Agent")?"diffValue:display,writeToDB ":""). (($hash->{ROLE} ne "Agent")?"index:list_all,recreate_Search_Idx,drop_Search_Idx,recreate_Report_Idx,drop_Report_Idx ":""). - (($dbmodel =~ /MYSQL/)?"adminCredentials ":""). + (($hash->{ROLE} ne "Agent" && $dbmodel =~ /MYSQL/)?"adminCredentials ":""). (($hash->{ROLE} ne "Agent")?"insert ":""). (($hash->{ROLE} ne "Agent")?"reduceLog ":""). (($hash->{ROLE} ne "Agent")?"sqlCmd:textField-long ":""). @@ -1043,7 +1052,7 @@ sub DbRep_Get($@) { if($success) { $atxt = "Username: $admusername, Password: $admpassword\n"; } else { - $atxt = "Credentials of $name couldn't be read - make sure you've set it with \"set $name adminCredentials username password\""; + $atxt = "Credentials of $name couldn't be read. Make sure you've set it with \"set $name adminCredentials username password\" (only valid for DbRep device type \"Client\")"; } return "Stored Credentials for database default access:\n". @@ -1191,6 +1200,7 @@ sub DbRep_Attr($$$$) { timeDiffToNow timeOlderThan sqlResultFormat + useAdminCredentials ); if ($aName eq "disable") { @@ -1450,7 +1460,7 @@ sub DbRep_Notify($$) { my $strucChanged; # altes in neues device in der DEF des angeschlossenen DbLog-device ändern (neues device loggen) - my $dblog_name = $own_hash->{dbloghash}{NAME}; # Name des an den DbRep-Agenten angeschlossenen DbLog-Dev + my $dblog_name = $own_hash->{HELPER}{DBLOGDEVICE}; # Name des an den DbRep-Agenten angeschlossenen DbLog-Dev my $dblog_hash = $defs{$dblog_name}; if ( $dblog_hash->{DEF} =~ m/( |\(|\|)$evl[1]( |\)|\||:)/ ) { @@ -2717,7 +2727,7 @@ sub averval_DoParse($) { my $st = [gettimeofday]; # DB-Abfrage zeilenweise für jeden Array-Eintrag - my $arrstr; + my ($arrstr,$wrstr,@rsf,@rsn); foreach my $row (@ts) { my @a = split("#", $row); my $runtime_string = $a[0]; @@ -2747,13 +2757,17 @@ sub averval_DoParse($) { my @line = $sth->fetchrow_array(); Log3 ($name, 5, "DbRep $name - SQL result: $line[0]") if($line[0]); - + if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); - $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."|"; + @rsf = split(/[ :]/,$runtime_string_first); + @rsn = split(/[ :]/,$runtime_string_next); + $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."|"; + $wrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."#".$rsn[0]."_".$rsn[1]."|"; # Kombi zum Rückschreiben in die DB } else { - my @rsf = split(" ",$runtime_string_first); - $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."|"; + @rsf = split(" ",$runtime_string_first); + @rsn = split(" ",$runtime_string_next); + $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."|"; + $wrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."#".$rsn[0]."|"; # Kombi zum Rückschreiben in die DB } } elsif ($acf eq "avgDailyMeanGWS") { @@ -2803,14 +2817,18 @@ sub averval_DoParse($) { } else { $sum = "insufficient values"; } - + if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); - $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."|"; + @rsf = split(/[ :]/,$runtime_string_first); + @rsn = split(/[ :]/,$runtime_string_next); + $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."|"; + $wrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."#".$rsn[0]."_".$rsn[1]."|"; # Kombi zum Rückschreiben in die DB } else { - my @rsf = split(" ",$runtime_string_first); - $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."|"; - } + @rsf = split(" ",$runtime_string_first); + @rsn = split(" ",$runtime_string_next); + $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."|"; + $wrstr .= $runtime_string."#".$sum."#".$rsf[0]."#".$rsn[0]."|"; # Kombi zum Rückschreiben in die DB + } } elsif ($acf eq "avgTimeWeightMean") { # zeitgewichteten Mittelwert berechnen @@ -2887,13 +2905,17 @@ sub averval_DoParse($) { Log3 ($name, 5, "DbRep $name - data element: $twmrow"); Log3 ($name, 5, "DbRep $name - time sum: $tsum, delta time: $dt, value: $val1, twm: ".$val1*($dt/$tsum)); } - } + } if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); - $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."|"; + @rsf = split(/[ :]/,$runtime_string_first); + @rsn = split(/[ :]/,$runtime_string_next); + $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."|"; + $wrstr .= $runtime_string."#".$sum."#".$rsf[0]."_".$rsf[1]."#".$rsn[0]."_".$rsn[1]."|"; # Kombi zum Rückschreiben in die DB } else { - my @rsf = split(" ",$runtime_string_first); - $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."|"; + @rsf = split(" ",$runtime_string_first); + @rsn = split(" ",$runtime_string_next); + $arrstr .= $runtime_string."#".$sum."#".$rsf[0]."|"; + $wrstr .= $runtime_string."#".$sum."#".$rsf[0]."#".$rsn[0]."|"; # Kombi zum Rückschreiben in die DB } } } @@ -2907,7 +2929,7 @@ sub averval_DoParse($) { # Ergebnisse in Datenbank schreiben my ($wrt,$irowdone); if($prop =~ /writeToDB/) { - ($wrt,$irowdone,$err) = DbRep_OutputWriteToDB($name,$device,$reading,$arrstr,$qlf); + ($wrt,$irowdone,$err) = DbRep_OutputWriteToDB($name,$device,$reading,$wrstr,$qlf); if ($err) { Log3 $hash->{NAME}, 2, "DbRep $name - $err"; $err = encode_base64($err,""); @@ -3251,7 +3273,7 @@ sub maxval_DoParse($) { if(!@array) { if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); + my @rsf = split(/[ :]/,$runtime_string_first); @array = ($runtime_string." "."0"." ".$rsf[0]."_".$rsf[1]."\n"); } else { my @rsf = split(" ",$runtime_string_first); @@ -3478,11 +3500,11 @@ sub minval_DoParse($) { return "$name|''|$device|$reading|''|$err|''"; } - my @array= map { $runtime_string." ".$_ -> [0]." ".$_ -> [1]."\n" } @{ $sth->fetchall_arrayref() }; + my @array = map { $runtime_string." ".$_->[0]." ".$_->[1]."\n" } @{ $sth->fetchall_arrayref() }; if(!@array) { if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); + my @rsf = split(/[ :]/,$runtime_string_first); @array = ($runtime_string." "."0"." ".$rsf[0]."_".$rsf[1]."\n"); } else { my @rsf = split(" ",$runtime_string_first); @@ -3521,7 +3543,6 @@ sub minval_DoParse($) { # Test auf $value = "numeric" if (!looks_like_number($value)) { - # $a[-1] =~ s/\s+$//g; Log3 ($name, 2, "DbRep $name - ERROR - value isn't numeric in minValue function. Faulty dataset was \nTIMESTAMP: $timestamp, DEVICE: $device, READING: $reading, VALUE: $value."); $err = encode_base64("Value isn't numeric. Faulty dataset was - TIMESTAMP: $timestamp, VALUE: $value", ""); return "$name|''|$device|$reading|''|$err|''"; @@ -3533,15 +3554,15 @@ sub minval_DoParse($) { if ($runtime_string eq $lastruntimestring) { if (!defined($min_value) || $value < $min_value) { - $min_value = $value; - $row_min_time = $timestamp; + $min_value = $value; + $row_min_time = $timestamp; $rh{$runtime_string} = $runtime_string."|".$min_value."|".$row_min_time; } } else { # neuer Zeitabschnitt beginnt, ersten Value-Wert erfassen - $lastruntimestring = $runtime_string; - $min_value = $value; - $row_min_time = $timestamp; + $lastruntimestring = $runtime_string; + $min_value = $value; + $row_min_time = $timestamp; $rh{$runtime_string} = $runtime_string."|".$min_value."|".$row_min_time; } $i++; @@ -3760,11 +3781,11 @@ sub diffval_DoParse($) { if(!@array) { if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); - @array = ($runtime_string." ".$rsf[0]."_".$rsf[1]."\n"); + my @rsf = split(/[ :]/,$runtime_string_first); + @array = ($runtime_string." ".$rsf[0]."_".$rsf[1]."\n"); } else { my @rsf = split(" ",$runtime_string_first); - @array = ($runtime_string." ".$rsf[0]."\n"); + @array = ($runtime_string." ".$rsf[0]."\n"); } } push(@row_array, @array); @@ -3802,13 +3823,7 @@ sub diffval_DoParse($) { $lastruntimestring = $runtime_string if ($i == 1); my $timestamp = $a[2]?$a[1]."_".$a[2]:$a[1]; my $value = $a[3]?$a[3]:0; - my $diff = $a[4]?sprintf("%.4f",$a[4]):0; - -# if ($uediff) { -# $diff = $diff + $uediff; -# Log3 ($name, 4, "DbRep $name - balance difference of $uediff between $rslval and $runtime_string"); -# $uediff = 0; -# } + my $diff = $a[4]?sprintf("%.4f",$a[4]):0; # Leerzeichen am Ende $timestamp entfernen $timestamp =~ s/\s+$//g; @@ -3883,7 +3898,7 @@ sub diffval_DoParse($) { foreach my $key (sort(keys%{$ncp})) { Log3 ($name, 3, $key) ; } - $ncps = join('§', %$ncp); + $ncps = join('§', %$ncp); $ncpslist = encode_base64($ncps,""); } @@ -4057,7 +4072,7 @@ sub sumval_DoParse($) { my $st = [gettimeofday]; # DB-Abfrage zeilenweise für jeden Array-Eintrag - my $arrstr; + my ($arrstr,$wrstr,@rsf,@rsn); foreach my $row (@ts) { my @a = split("#", $row); my $runtime_string = $a[0]; @@ -4087,11 +4102,15 @@ sub sumval_DoParse($) { Log3 ($name, 5, "DbRep $name - SQL result: $line[0]") if($line[0]); if(AttrVal($name, "aggregation", "") eq "hour") { - my @rsf = split(/[" "\|":"]/,$runtime_string_first); - $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."|"; + @rsf = split(/[ :]/,$runtime_string_first); + @rsn = split(/[ :]/,$runtime_string_next); + $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."|"; + $wrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."_".$rsf[1]."#".$rsn[0]."_".$rsn[1]."|"; # Kombi zum Rückschreiben in die DB } else { - my @rsf = split(" ",$runtime_string_first); - $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."|"; + @rsf = split(" ",$runtime_string_first); + @rsn = split(" ",$runtime_string_next); + $arrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."|"; + $wrstr .= $runtime_string."#".$line[0]."#".$rsf[0]."#".$rsn[0]."|"; # Kombi zum Rückschreiben in die DB } } @@ -4104,7 +4123,7 @@ sub sumval_DoParse($) { # Ergebnisse in Datenbank schreiben my ($wrt,$irowdone); if($prop =~ /writeToDB/) { - ($wrt,$irowdone,$err) = DbRep_OutputWriteToDB($name,$device,$reading,$arrstr,"sum"); + ($wrt,$irowdone,$err) = DbRep_OutputWriteToDB($name,$device,$reading,$wrstr,"sum"); if ($err) { Log3 $hash->{NAME}, 2, "DbRep $name - $err"; $err = encode_base64($err,""); @@ -5162,7 +5181,7 @@ sub deldoublets_DoParse($) { my $dbpassword = $attr{"sec$dblogname"}{secret}; my $utf8 = defined($hash->{UTF8})?$hash->{UTF8}:0; my $limit = AttrVal($name, "limit", 1000); - my ($err,$dbh,$sth,$sql,$rowlist,$selspec,$st,$table,$addon); + my ($err,$dbh,$sth,$sql,$rowlist,$selspec,$st,$table,$addon,$dsql); # Background-Startzeit my $bst = [gettimeofday]; @@ -5227,18 +5246,19 @@ sub deldoublets_DoParse($) { # SQL-Laufzeit ermitteln $rt = $rt+tv_interval($st); - # Beginn Löschlogik, Zusammenstellen der löschenden DS (warping) + # Beginn Löschlogik, Zusammenstellen der zu löschenden DS (warping) # Array @warp -> die zu löschenden Datensätze my (@warp); my $i = 0; foreach my $nr (map { $_->[1]."_ESC_".$_->[2]."_ESC_".($_->[0] =~ s/ /_ESC_/r)."_ESC_".$_->[3]."_|_".($_->[4]-1) } @{$sth->fetchall_arrayref()}) { # Reihenfolge geändert in: DEVICE,READING,DATE,TIME,VALUE,count(*) if($opt =~ /adviceDelete/) { - push(@warp,$nr) if($#todel+1 < $limit); # die zu löschenden Datensätze (nur zur Anzeige) + push(@warp,$i."_".$nr) if($#todel+1 < $limit); # die zu löschenden Datensätze (nur zur Anzeige) } else { - push (@warp,$nr); # Array der zu löschenden Datensätze + push(@warp,$i."_".$nr); # Array der zu löschenden Datensätze } - my $c = (split("|",$nr))[-1]; + my $c = (split("|",$nr))[-1]; + Log3 ($name, 4, "DbRep $name - WARP: $nr, ntodel: $ntodel, c: $c"); $ntodel = $ntodel + $c; if ($opt =~ /delete/) { # delete Datensätze @@ -5250,9 +5270,12 @@ sub deldoublets_DoParse($) { $val =~ s/'/''/g; # escape ' with '' $val =~ s/\\/\\\\/g if($model eq "MYSQL"); # escape \ with \\ für MySQL $st = [gettimeofday]; - my $dsql = "delete FROM $table WHERE TIMESTAMP = '$dt' AND DEVICE = '$dev' AND READING = '$read' AND VALUE = '$val' limit $limit;"; - my $sthd = $dbh->prepare($dsql); - Log3 ($name, 4, "DbRep $name - SQL execute: $dsql"); + if($model =~ /MYSQL|SQLITE/) { + $dsql = "delete FROM $table WHERE TIMESTAMP = '$dt' AND DEVICE = '$dev' AND READING = '$read' AND VALUE = '$val' limit $limit;"; + } elsif ($model eq "POSTGRESQL") { + $dsql = "DELETE FROM $table WHERE ctid = any (array(SELECT ctid FROM $table WHERE TIMESTAMP = '$dt' AND DEVICE = '$dev' AND READING = '$read' AND VALUE = '$val' ORDER BY timestamp LIMIT $limit));"; + } + my $sthd = $dbh->prepare($dsql); eval {$sthd->execute();}; if ($@) { @@ -10612,7 +10635,7 @@ return \%ncp; # Funktionsergebnisse in Datenbank schreiben #################################################################################################### sub DbRep_OutputWriteToDB($$$$$) { - my ($name,$device,$reading,$arrstr,$optxt) = @_; + my ($name,$device,$reading,$wrstr,$optxt) = @_; my $hash = $defs{$name}; my $dbloghash = $defs{$hash->{HELPER}{DBLOGDEVICE}}; my $dbconn = $dbloghash->{dbconn}; @@ -10630,7 +10653,8 @@ sub DbRep_OutputWriteToDB($$$$$) { my $unit = ""; my $wrt = 0; my $irowdone = 0; - my ($dbh,$sth_ih,$sth_uh,$sth_ic,$sth_uc,$err,$timestamp,$value,$date,$time,$rsf,$aggr,@row_array); + my ($dbh,$sth_ih,$sth_uh,$sth_ic,$sth_uc,$err,$timestamp,$value,$date,$time,$hour,$ndate,$ntime,$rsf,$rsn,$aggr,@row_array); + my ($year,$mon,$mday,$t1,$corr); if(!$dbloghash->{HELPER}{COLSET}) { $err = "No result of \"$hash->{LASTCMD}\" to database written. Cause: column width in \"$hash->{DEF}\" isn't set"; @@ -10644,32 +10668,65 @@ sub DbRep_OutputWriteToDB($$$$$) { $type = $defs{$device}{TYPE} if($defs{$device}); # $type vom Device ableiten if($optxt =~ /avg|sum/) { - my @arr = split("\\|", $arrstr); + my @arr = split("\\|", $wrstr); + my $ele = $#arr; # Nr des letzten Elements + my $i = 0; foreach my $row (@arr) { my @a = split("#", $row); my $runtime_string = $a[0]; # Aggregations-Alias (nicht benötigt) $value = defined($a[1])?sprintf("%.4f",$a[1]):undef; - $rsf = $a[2]; # Datum / Zeit für DB-Speicherung + $rsf = $a[2]; # Runtime String first - Datum / Zeit für DB-Speicherung ($date,$time) = split("_",$rsf); $time =~ s/-/:/g if($time); + $rsn = $a[3]; # Runtime String next - Datum / Zeit für DB-Speicherung + ($ndate,$ntime) = split("_",$rsn); + $ntime =~ s/-/:/g if($ntime); if($time !~ /^(\d{2}):(\d{2}):(\d{2})$/) { - if($aggr =~ /no|day|week|month/) { - $time = "23:59:58"; + if($aggr =~ /no|day|week|month|year/) { + $time = "00:00:01"; # https://forum.fhem.de/index.php/topic,105787.msg1013920.html#msg1013920 + $ntime = "23:59:59"; + ($year,$mon,$mday) = split("-", $ndate); + $corr = ($i != $ele) ? 86400 : 0; + $t1 = fhemTimeLocal(59, 59, 23, $mday, $mon-1, $year-1900)-$corr; + ($ndate,undef) = split(" ",FmtDateTime($t1)); } elsif ($aggr =~ /hour/) { - $time = "$time:59:58"; + $hour = $time; + $time = "$hour:00:01"; # https://forum.fhem.de/index.php/topic,105787.msg1013920.html#msg1013920 + $ntime = "$hour:59:59"; + if ($ntime eq "23:59:59") { + ($year,$mon,$mday) = split("-", $ndate); + $t1 = fhemTimeLocal(59, 59, 23, $mday, $mon-1, $year-1900)-86400; + ($ndate,undef) = split(" ",FmtDateTime($t1)); + } } } - if ($value) { + if (defined $value) { # Daten auf maximale Länge beschneiden (DbLog-Funktion !) ($device,$type,$event,$reading,$value,$unit) = DbLog_cutCol($dbloghash,$device,$type,$event,$reading,$value,$unit); - push(@row_array, "$date $time|$device|$type|$event|$reading|$value|$unit"); + if($i == 0) { + push(@row_array, "$date $time|$device|$type|$event|$reading|$value|$unit"); + push(@row_array, "$ndate $ntime|$device|$type|$event|$reading|$value|$unit"); + } else { + if ($aggr =~ /no|day|week|month|year/) { + ($year,$mon,$mday) = split("-", $date); + $t1 = fhemTimeLocal(01, 00, 00, $mday, $mon-1, $year-1900); + ($date,$time) = split(" ",FmtDateTime($t1)); + } elsif ($aggr =~ /hour/) { + ($year,$mon,$mday) = split("-", $date); + $t1 = fhemTimeLocal(01, 00, $hour, $mday, $mon-1, $year-1900); + ($date,$time) = split(" ",FmtDateTime($t1)); + } + push(@row_array, "$date $time|$device|$type|$event|$reading|$value|$unit"); + push(@row_array, "$ndate $ntime|$device|$type|$event|$reading|$value|$unit"); + } } + $i++; } } if($optxt =~ /min|max|diff/) { - my %rh = split("§", $arrstr); + my %rh = split("§", $wrstr); foreach my $key (sort(keys(%rh))) { my @k = split("\\|",$rh{$key}); $rsf = $k[2]; # Datum / Zeit für DB-Speicherung @@ -10690,7 +10747,7 @@ sub DbRep_OutputWriteToDB($$$$$) { push(@row_array, "$date $time|$device|$type|$event|$reading|$value|$unit"); } } - } + } if (@row_array) { # Schreibzyklus aktivieren @@ -10710,7 +10767,7 @@ sub DbRep_OutputWriteToDB($$$$$) { } if (lc($DbLogType) =~ m(history)) { - # insert history mit/ohne primary key + # INSERT history mit/ohne primary key if ($usepkh && $dbloghash->{MODEL} eq 'MYSQL') { eval { $sth_ih = $dbh->prepare_cached("INSERT IGNORE INTO history (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); }; } elsif ($usepkh && $dbloghash->{MODEL} eq 'SQLITE') { @@ -10725,22 +10782,10 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3 ($name, 2, "DbRep $name - $@"); return ($wrt,$irowdone,$err); } - # update history mit/ohne primary key - if ($usepkh && $hash->{MODEL} eq 'MYSQL') { - $sth_uh = $dbh->prepare("REPLACE INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkh && $hash->{MODEL} eq 'SQLITE') { - $sth_uh = $dbh->prepare("INSERT OR REPLACE INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkh && $hash->{MODEL} eq 'POSTGRESQL') { - $sth_uh = $dbh->prepare("INSERT INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?) ON CONFLICT ($pkc) - DO UPDATE SET TIMESTAMP=EXCLUDED.TIMESTAMP, DEVICE=EXCLUDED.DEVICE, TYPE=EXCLUDED.TYPE, EVENT=EXCLUDED.EVENT, READING=EXCLUDED.READING, - VALUE=EXCLUDED.VALUE, UNIT=EXCLUDED.UNIT"); - } else { - $sth_uh = $dbh->prepare("UPDATE history SET TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (TIMESTAMP=?) AND (DEVICE=?) AND (READING=?)"); - } } if (lc($DbLogType) =~ m(current) ) { - # insert current mit/ohne primary key + # INSERT current mit/ohne primary key if ($usepkc && $hash->{MODEL} eq 'MYSQL') { eval { $sth_ic = $dbh->prepare("INSERT IGNORE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); }; } elsif ($usepkc && $hash->{MODEL} eq 'SQLITE') { @@ -10756,18 +10801,6 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3 ($name, 2, "DbRep $name - $@"); return ($wrt,$irowdone,$err); } - # update current mit/ohne primary key - if ($usepkc && $hash->{MODEL} eq 'MYSQL') { - $sth_uc = $dbh->prepare("REPLACE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkc && $hash->{MODEL} eq 'SQLITE') { - $sth_uc = $dbh->prepare("INSERT OR REPLACE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkc && $hash->{MODEL} eq 'POSTGRESQL') { - $sth_uc = $dbh->prepare("INSERT INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?) ON CONFLICT ($pkc) - DO UPDATE SET TIMESTAMP=EXCLUDED.TIMESTAMP, DEVICE=EXCLUDED.DEVICE, TYPE=EXCLUDED.TYPE, EVENT=EXCLUDED.EVENT, READING=EXCLUDED.READING, - VALUE=EXCLUDED.VALUE, UNIT=EXCLUDED.UNIT"); - } else { - $sth_uc = $dbh->prepare("UPDATE current SET TIMESTAMP=?, TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (DEVICE=?) AND (READING=?)"); - } } eval { $dbh->begin_work() if($dbh->{AutoCommit}); }; @@ -10775,8 +10808,6 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3($name, 2, "DbRep $name -> Error start transaction for history - $@"); } - Log3 $hash->{NAME}, 4, "DbRep $name - data prepared to db write:"; - # SQL-Startzeit my $wst = [gettimeofday]; @@ -10791,46 +10822,44 @@ sub DbRep_OutputWriteToDB($$$$$) { $reading = $a[4]; $value = $a[5]; $unit = $a[6]; - Log3 $hash->{NAME}, 4, "DbRep $name - $row"; eval { # update oder insert history if (lc($DbLogType) =~ m(history) ) { - my $rv_uh = $sth_uh->execute($type,$event,$value,$unit,$timestamp,$device,$reading); + my $rv_uh = $dbh->do("UPDATE history SET TIMESTAMP=\"$timestamp\", DEVICE=\"$device\", READING=\"$reading\", TYPE=\"$type\", EVENT=\"$event\", VALUE=\"$value\", UNIT=\"$unit\" WHERE TIMESTAMP=\"$timestamp\" AND DEVICE=\"$device\" AND READING=\"$reading\""); + $uhs += $rv_uh if($rv_uh); + Log3 $hash->{NAME}, 4, "DbRep $name - UPDATE history: $row, RESULT: $rv_uh"; if ($rv_uh == 0) { - $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); - $ihs++; - } else { - $uhs++; + my $rv_ih = $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); + $ihs += $rv_ih if($rv_ih); + Log3 $hash->{NAME}, 4, "DbRep $name - INSERT history: $row, RESULT: $rv_ih"; } } # update oder insert current if (lc($DbLogType) =~ m(current) ) { - my $rv_uc = $sth_uc->execute($timestamp,$type,$event,$value,$unit,$device,$reading); + my $rv_uc = $dbh->do("UPDATE current SET TIMESTAMP=\"$timestamp\", DEVICE=\"$device\", READING=\"$reading\", TYPE=\"$type\", EVENT=\"$event\", VALUE=\"$value\", UNIT=\"$unit\" WHERE DEVICE=\"$device\" AND READING=\"$reading\""); if ($rv_uc == 0) { $sth_ic->execute($timestamp,$device,$type,$event,$reading,$value,$unit); } - } + } }; - + if ($@) { $err = $@; Log3 ($name, 2, "DbRep $name - $@"); $dbh->rollback; $dbh->disconnect; - $ihs = 0; - $uhs = 0; return ($wrt,0,$err); - } else { - $irowdone++; - } + } } eval {$dbh->commit() if(!$dbh->{AutoCommit});}; + $dbh->disconnect; Log3 $hash->{NAME}, 3, "DbRep $name - number of lines updated in \"$dblogname\": $uhs"; Log3 $hash->{NAME}, 3, "DbRep $name - number of lines inserted into \"$dblogname\": $ihs"; + $irowdone = $ihs + $uhs; # SQL-Laufzeit ermitteln $wrt = tv_interval($wst); @@ -10853,6 +10882,7 @@ sub DbRep_WriteToDB($$$@) { my $supk = AttrVal($dblogname, "noSupportPK", 0); my $wrt = 0; my $irowdone = 0; + my $totrows = 0; my ($sth_ih,$sth_uh,$sth_ic,$sth_uc,$err); # check ob PK verwendet wird, @usepkx?Anzahl der Felder im PK:0 wenn kein PK, $pkx?Namen der Felder:none wenn kein PK @@ -10934,8 +10964,7 @@ sub DbRep_WriteToDB($$$@) { # SQL-Startzeit my $wst = [gettimeofday]; - my $ihs = 0; - my $uhs = 0; + my ($ihs,$uhs) = (0,0); foreach my $row (@row_array) { my ($date,$time,$device,$type,$event,$reading,$value,$unit) = ($row =~ /^(.*)_ESC_(.*)_ESC_(.*)_ESC_(.*)_ESC_(.*)_ESC_(.*)_ESC_(.*)_ESC_(.*)$/); Log3 $hash->{NAME}, 5, "DbRep $name - $row"; @@ -10949,10 +10978,10 @@ sub DbRep_WriteToDB($$$@) { $rv_uh = $sth_uh->execute($type,$event,$value,$unit,$timestamp,$device,$reading); } if ($rv_uh == 0) { - $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); - $ihs++; + $ihs += $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); # V8.30.7 + # $ihs++; # V8.30.7 } else { - $uhs++; + $uhs += $rv_uh; # V8.30.7 } } # update oder insert current @@ -10968,18 +10997,18 @@ sub DbRep_WriteToDB($$$@) { $err = $@; Log3 ($name, 2, "DbRep $name - $@"); $dbh->rollback; - $ihs = 0; - $uhs = 0; return ($wrt,0,$err); } else { - $irowdone++; - } + $totrows++; # V8.30.7 + } } eval {$dbh->commit() if(!$dbh->{AutoCommit});}; - - Log3 $hash->{NAME}, 3, "DbRep $name - number of lines updated in \"$dblogname\": $uhs" if($uhs); - Log3 $hash->{NAME}, 3, "DbRep $name - number of lines inserted into \"$dblogname\": $ihs" if($ihs); + + Log3 ($name, 3, "DbRep $name - total lines transfered to standby database: $totrows"); # V8.30.7 + Log3 ($name, 3, "DbRep $name - number of lines updated in \"$dblogname\": $uhs") if($uhs); + Log3 ($name, 3, "DbRep $name - number of lines inserted into \"$dblogname\": $ihs") if($ihs); + $irowdone = $ihs + $uhs; # SQL-Laufzeit ermitteln $wrt = tv_interval($wst); @@ -11007,8 +11036,8 @@ sub DbRep_checkUsePK ($$$){ $pkc =~ tr/"//d; $upkh = 1 if(@pkh && @pkh ne "none"); $upkc = 1 if(@pkc && @pkc ne "none"); - Log3 $hash->{NAME}, 5, "DbRep $name -> Primary Key used in $db.history: $pkh"; - Log3 $hash->{NAME}, 5, "DbRep $name -> Primary Key used in $db.current: $pkc"; + Log3 $hash->{NAME}, 5, "DbRep $name -> Primary Key used in $db.history: $upkh ($pkh)"; + Log3 $hash->{NAME}, 5, "DbRep $name -> Primary Key used in $db.current: $upkc ($pkc)"; return ($upkh,$upkc,$pkh,$pkc); } @@ -11569,7 +11598,7 @@ return; - Save a user / password for the privileged respectively administrative database access. The user is required for database operations which has to be executed by a privileged user. Please see also attribute 'useAdminCredentials'.
- (only valid if database type is MYSQL) + (only valid if database type is MYSQL and DbRep-type "Client")
@@ -13723,7 +13752,7 @@ sub bdump {
  • useAdminCredentials - If set, a before with "set <aame> adminCredentials" saved privileged user is used for particular database operations.
    - (only valid if database type is MYSQL) + (only valid if database type is MYSQL and DbRep-type "Client")

    • @@ -14057,7 +14086,7 @@ sub bdump { - Speichert einen User / Passwort für den privilegierten bzw. administrativen Datenbankzugriff. Er wird bei Datenbankoperationen benötigt, die mit einem privilegierten User ausgeführt werden müssen. Siehe auch Attribut 'useAdminCredentials'.
    - (nur gültig bei Datenbanktyp MYSQL) + (nur gültig bei Datenbanktyp MYSQL und DbRep-Typ "Client")
    @@ -16251,7 +16280,7 @@ sub bdump {
  • useAdminCredentials - Wenn gesetzt, wird ein zuvor mit "set <Name> adminCredentials" gespeicherter privilegierter User für bestimmte Datenbankoperationen verwendet.
    - (nur gültig für Datenbanktyp MYSQL) + (nur gültig für Datenbanktyp MYSQL und DbRep-Typ "Client")

    • @@ -16479,4 +16508,4 @@ sub bdump { } =end :application/json;q=META.json -=cut \ No newline at end of file +=cutt \ No newline at end of file diff --git a/fhem/contrib/DS_Starter/93_DbRep.pm b/fhem/contrib/DS_Starter/93_DbRep.pm index 6ed723f28..c52665bc3 100644 --- a/fhem/contrib/DS_Starter/93_DbRep.pm +++ b/fhem/contrib/DS_Starter/93_DbRep.pm @@ -58,6 +58,7 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # Version History intern our %DbRep_vNotesIntern = ( + "8.30.8" => "25.01.2020 adjust SQL-Statements in OutputWriteToDB to avoid Duplicate entry errors and other fixes ", "8.30.7" => "24.01.2020 corrected count of inserted rows into standby database (DbRep_WriteToDB) ", "8.30.6" => "23.01.2020 delDoublets now are working also for PostgreSQL, calculation of number_fetched_rows corrected ", "8.30.5" => "23.01.2020 remove adminCredentials from set of device type \"Agent\" ", @@ -2074,7 +2075,7 @@ sub DbRep_createTimeArray($$$) { # absolute Auswertungszeiträume statische und dynamische (Beginn / Ende) berechnen if($hash->{HELPER}{MINTS} && $hash->{HELPER}{MINTS} =~ m/0000-00-00/) { - Log3 ($name, 1, "DbRep $name - ERROR - wrong timestamp \"$hash->{HELPER}{MINTS}\" found in database. Please delete it with 'set $name sqlCmd delete from history where TIMESTAMP=\"0000-00-00 00:00:00\";' "); + Log3 ($name, 1, "DbRep $name - ERROR - wrong timestamp \"$hash->{HELPER}{MINTS}\" found in database. Please delete it !"); delete $hash->{HELPER}{MINTS}; } @@ -10766,7 +10767,7 @@ sub DbRep_OutputWriteToDB($$$$$) { } if (lc($DbLogType) =~ m(history)) { - # insert history mit/ohne primary key + # INSERT history mit/ohne primary key if ($usepkh && $dbloghash->{MODEL} eq 'MYSQL') { eval { $sth_ih = $dbh->prepare_cached("INSERT IGNORE INTO history (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); }; } elsif ($usepkh && $dbloghash->{MODEL} eq 'SQLITE') { @@ -10781,22 +10782,10 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3 ($name, 2, "DbRep $name - $@"); return ($wrt,$irowdone,$err); } - # update history mit/ohne primary key - if ($usepkh && $hash->{MODEL} eq 'MYSQL') { - $sth_uh = $dbh->prepare("REPLACE INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkh && $hash->{MODEL} eq 'SQLITE') { - $sth_uh = $dbh->prepare("INSERT OR REPLACE INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkh && $hash->{MODEL} eq 'POSTGRESQL') { - $sth_uh = $dbh->prepare("INSERT INTO history (TYPE, EVENT, VALUE, UNIT, TIMESTAMP, DEVICE, READING) VALUES (?,?,?,?,?,?,?) ON CONFLICT ($pkc) - DO UPDATE SET TIMESTAMP=EXCLUDED.TIMESTAMP, DEVICE=EXCLUDED.DEVICE, TYPE=EXCLUDED.TYPE, EVENT=EXCLUDED.EVENT, READING=EXCLUDED.READING, - VALUE=EXCLUDED.VALUE, UNIT=EXCLUDED.UNIT"); - } else { - $sth_uh = $dbh->prepare("UPDATE history SET TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (TIMESTAMP=?) AND (DEVICE=?) AND (READING=?)"); - } } if (lc($DbLogType) =~ m(current) ) { - # insert current mit/ohne primary key + # INSERT current mit/ohne primary key if ($usepkc && $hash->{MODEL} eq 'MYSQL') { eval { $sth_ic = $dbh->prepare("INSERT IGNORE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); }; } elsif ($usepkc && $hash->{MODEL} eq 'SQLITE') { @@ -10812,18 +10801,6 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3 ($name, 2, "DbRep $name - $@"); return ($wrt,$irowdone,$err); } - # update current mit/ohne primary key - if ($usepkc && $hash->{MODEL} eq 'MYSQL') { - $sth_uc = $dbh->prepare("REPLACE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkc && $hash->{MODEL} eq 'SQLITE') { - $sth_uc = $dbh->prepare("INSERT OR REPLACE INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?)"); - } elsif ($usepkc && $hash->{MODEL} eq 'POSTGRESQL') { - $sth_uc = $dbh->prepare("INSERT INTO current (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES (?,?,?,?,?,?,?) ON CONFLICT ($pkc) - DO UPDATE SET TIMESTAMP=EXCLUDED.TIMESTAMP, DEVICE=EXCLUDED.DEVICE, TYPE=EXCLUDED.TYPE, EVENT=EXCLUDED.EVENT, READING=EXCLUDED.READING, - VALUE=EXCLUDED.VALUE, UNIT=EXCLUDED.UNIT"); - } else { - $sth_uc = $dbh->prepare("UPDATE current SET TIMESTAMP=?, TYPE=?, EVENT=?, VALUE=?, UNIT=? WHERE (DEVICE=?) AND (READING=?)"); - } } eval { $dbh->begin_work() if($dbh->{AutoCommit}); }; @@ -10831,8 +10808,6 @@ sub DbRep_OutputWriteToDB($$$$$) { Log3($name, 2, "DbRep $name -> Error start transaction for history - $@"); } - Log3 $hash->{NAME}, 4, "DbRep $name - data prepared to db write:"; - # SQL-Startzeit my $wst = [gettimeofday]; @@ -10847,46 +10822,44 @@ sub DbRep_OutputWriteToDB($$$$$) { $reading = $a[4]; $value = $a[5]; $unit = $a[6]; - Log3 $hash->{NAME}, 4, "DbRep $name - $row"; eval { # update oder insert history if (lc($DbLogType) =~ m(history) ) { - my $rv_uh = $sth_uh->execute($type,$event,$value,$unit,$timestamp,$device,$reading); + my $rv_uh = $dbh->do("UPDATE history SET TIMESTAMP=\"$timestamp\", DEVICE=\"$device\", READING=\"$reading\", TYPE=\"$type\", EVENT=\"$event\", VALUE=\"$value\", UNIT=\"$unit\" WHERE TIMESTAMP=\"$timestamp\" AND DEVICE=\"$device\" AND READING=\"$reading\""); + $uhs += $rv_uh if($rv_uh); + Log3 $hash->{NAME}, 4, "DbRep $name - UPDATE history: $row, RESULT: $rv_uh"; if ($rv_uh == 0) { - $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); - $ihs++; - } else { - $uhs++; + my $rv_ih = $sth_ih->execute($timestamp,$device,$type,$event,$reading,$value,$unit); + $ihs += $rv_ih if($rv_ih); + Log3 $hash->{NAME}, 4, "DbRep $name - INSERT history: $row, RESULT: $rv_ih"; } } # update oder insert current if (lc($DbLogType) =~ m(current) ) { - my $rv_uc = $sth_uc->execute($timestamp,$type,$event,$value,$unit,$device,$reading); + my $rv_uc = $dbh->do("UPDATE current SET TIMESTAMP=\"$timestamp\", DEVICE=\"$device\", READING=\"$reading\", TYPE=\"$type\", EVENT=\"$event\", VALUE=\"$value\", UNIT=\"$unit\" WHERE DEVICE=\"$device\" AND READING=\"$reading\""); if ($rv_uc == 0) { $sth_ic->execute($timestamp,$device,$type,$event,$reading,$value,$unit); } - } + } }; - + if ($@) { $err = $@; Log3 ($name, 2, "DbRep $name - $@"); $dbh->rollback; $dbh->disconnect; - $ihs = 0; - $uhs = 0; return ($wrt,0,$err); - } else { - $irowdone++; - } + } } eval {$dbh->commit() if(!$dbh->{AutoCommit});}; + $dbh->disconnect; Log3 $hash->{NAME}, 3, "DbRep $name - number of lines updated in \"$dblogname\": $uhs"; Log3 $hash->{NAME}, 3, "DbRep $name - number of lines inserted into \"$dblogname\": $ihs"; + $irowdone = $ihs + $uhs; # SQL-Laufzeit ermitteln $wrt = tv_interval($wst);