FHEM/fhem-mirror
2
0
Fork 0
mirror of https://github.com/fhem/fhem-mirror.git synced 2025-03-10 03:06:37 +00:00
Code Issues Releases Wiki Activity

93_DbRep: check index "Report_Idx" during first DB connect

Browse Source 
git-svn-id: https://svn.fhem.de/fhem/trunk@19274 2b470e98-0d58-463d-a4d8-8e2adae1ed80
This commit is contained in:
nasseeder1 2019-04-28 08:22:45 +00:00
parent 52715ac62e
commit ab795fe7a6
3 changed files with 145 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fhem/CHANGED
View File

@ -1,5 +1,6 @@
# Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Add changes at the top of the list. Keep it in ASCII, and 80-char wide.
# Do not insert empty lines here, update check depends on it. # Do not insert empty lines here, update check depends on it.
- change: 93_DbRep: check index "Report_Idx" during first DB connect
- change: 98_RandomTimer: remove 59_Twilight dependency - change: 98_RandomTimer: remove 59_Twilight dependency
- feature: 93_DbRep: new set "index" command to manage needed indexe for - feature: 93_DbRep: new set "index" command to manage needed indexe for
DbLog and DbRep, minor code changes DbLog and DbRep, minor code changes

110
fhem/FHEM/93_DbRep.pm
View File

@ -58,6 +58,7 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch';
# Version History intern # Version History intern
our %DbRep_vNotesIntern = ( our %DbRep_vNotesIntern = (
"8.20.1" => "28.04.2019 set index verbose changed, check index \"Report_Idx\" in getInitData ",
"8.20.0" => "27.04.2019 don't save hash refs in central hash to prevent potential memory leak, new set \"index\" ". "8.20.0" => "27.04.2019 don't save hash refs in central hash to prevent potential memory leak, new set \"index\" ".
"command, \"repository\" added in Meta.json ", "command, \"repository\" added in Meta.json ",
"8.19.1" => "10.04.2019 adjust \$hash->{HELPER}{IDRETRIES} if value is negative ", "8.19.1" => "10.04.2019 adjust \$hash->{HELPER}{IDRETRIES} if value is negative ",
@ -1397,8 +1398,7 @@ return undef;
################################################################################### ###################################################################################
# First Init DB Connect # First Init DB Connect
# Verbindung zur DB aufbauen und den Timestamp des ältesten # Verbindung zur DB aufbauen und Datenbankeigenschaften ermitteln
# Datensatzes ermitteln
################################################################################### ###################################################################################
sub DbRep_firstconnect(@) { sub DbRep_firstconnect(@) {
my ($string) = @_; my ($string) = @_;
@ -1439,54 +1439,85 @@ return;
} }
#################################################################################################### ####################################################################################################
# den ältesten Datensatz (Timestamp) in der DB bestimmen # DatenDatenbankeigenschaften ermitteln
#################################################################################################### ####################################################################################################
sub DbRep_getInitData($) { sub DbRep_getInitData($) {
my ($string) = @_; my ($string) = @_;
my ($name,$opt,$prop,$fret) = split("\\|", $string); my ($name,$opt,$prop,$fret) = split("\\|", $string);
my $hash = $defs{$name}; my $hash = $defs{$name};
my $dbloghash = $hash->{dbloghash}; my $dbloghash = $hash->{dbloghash};
my $dbconn = $dbloghash->{dbconn}; my $dbconn = $dbloghash->{dbconn};
my $dbuser = $dbloghash->{dbuser}; my $dbuser = $dbloghash->{dbuser};
my $dblogname = $dbloghash->{NAME}; my $dblogname = $dbloghash->{NAME};
my $dbpassword = $attr{"sec$dblogname"}{secret}; my $dbmodel = $dbloghash->{MODEL};
my $mintsdef = "1970-01-01 01:00:00"; my $dbpassword = $attr{"sec$dblogname"}{secret};
my ($dbh,$sql,$err,$mints); my $mintsdef = "1970-01-01 01:00:00";
my $idxstate = "";
my ($dbh,$sql,$err,$mints);
# Background-Startzeit # Background-Startzeit
my $bst = [gettimeofday]; my $bst = [gettimeofday];
eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1, AutoInactiveDestroy => 1 }); }; eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1, AutoInactiveDestroy => 1 }); };
if ($@) { if ($@) {
$err = encode_base64($@,""); $err = encode_base64($@,"");
Log3 ($name, 2, "DbRep $name - $@"); Log3 ($name, 2, "DbRep $name - $@");
return "$name|''|''|$err"; return "$name|''|''|$err";
} }
# SQL-Startzeit # SQL-Startzeit
my $st = [gettimeofday]; my $st = [gettimeofday];
eval { $mints = $dbh->selectrow_array("SELECT min(TIMESTAMP) FROM history;"); }; # ältesten Datensatz der DB ermitteln
eval { $mints = $dbh->selectrow_array("SELECT min(TIMESTAMP) FROM history;"); };
$dbh->disconnect; # Report_Idx Status ermitteln
my ($ava,$sqlava);
my $idx = "Report_Idx";
if($dbmodel =~ /MYSQL/) {
$sqlava = "SHOW INDEX FROM history where Key_name='$idx';";
} elsif($dbmodel =~ /SQLITE/) {
$sqlava = "SELECT name FROM sqlite_master WHERE type='index' AND name='$idx';";
} elsif($dbmodel =~ /POSTGRESQL/) {
$sqlava = "SELECT indexname FROM pg_indexes WHERE tablename='history' and indexname ='$idx';";
}
eval {$ava = $dbh->selectrow_array($sqlava)};
if($@) {
$idxstate = "state of Index $idx can't be determined !";
Log3($name, 2, "DbRep $name - WARNING - $idxstate");
} else {
if($hash->{LASTCMD} ne "minTimestamp") {
if($ava) {
$idxstate = "Index $idx exists";
Log3($name, 3, "DbRep $name - $idxstate. Check ok");
} else {
$idxstate = "Index $idx doesn't exist. Please create the index by \"set $name index recreate_Report_Idx\" command !";
Log3($name, 2, "DbRep $name - WARNING - $idxstate");
}
}
}
# SQL-Laufzeit ermitteln $dbh->disconnect;
my $rt = tv_interval($st);
$mints = $mints?encode_base64($mints,""):encode_base64($mintsdef,""); # SQL-Laufzeit ermitteln
my $rt = tv_interval($st);
# Background-Laufzeit ermitteln $mints = $mints?encode_base64($mints,""):encode_base64($mintsdef,"");
my $brt = tv_interval($bst); $idxstate = encode_base64($idxstate,"");
# Background-Laufzeit ermitteln
my $brt = tv_interval($bst);
$rt = $rt.",".$brt; $rt = $rt.",".$brt;
no warnings 'uninitialized'; no warnings 'uninitialized';
Log3 ($name, 3, "DbRep $name - Initial data information retrieved successfully - total time used: $brt seconds"); Log3 ($name, 3, "DbRep $name - Initial data information retrieved successfully - total time used: ".sprintf("%.4f",$brt)." seconds");
return "$name|$mints|$rt|0|$opt|$prop|$fret"; return "$name|$mints|$rt|0|$opt|$prop|$fret|$idxstate";
} }
#################################################################################################### ####################################################################################################
# Auswertungsroutine den ältesten Datensatz (Timestamp) in der DB bestimmen # Auswertungsroutine DbRep_getInitData
#################################################################################################### ####################################################################################################
sub DbRep_getInitDataDone($) { sub DbRep_getInitDataDone($) {
my ($string) = @_; my ($string) = @_;
@ -1500,6 +1531,7 @@ sub DbRep_getInitDataDone($) {
my $opt = $a[4]; my $opt = $a[4];
my $prop = $a[5]; my $prop = $a[5];
my $fret = \&{$a[6]} if($a[6]); my $fret = \&{$a[6]} if($a[6]);
my $idxstate = decode_base64($a[7]);
my $dblogdevice = $hash->{HELPER}{DBLOGDEVICE}; my $dblogdevice = $hash->{HELPER}{DBLOGDEVICE};
$hash->{dbloghash} = $defs{$dblogdevice}; $hash->{dbloghash} = $defs{$dblogdevice};
my $dbconn = $hash->{dbloghash}{dbconn}; my $dbconn = $hash->{dbloghash}{dbconn};
@ -1518,6 +1550,7 @@ sub DbRep_getInitDataDone($) {
readingsBeginUpdate($hash); readingsBeginUpdate($hash);
ReadingsBulkUpdateValue ($hash, "timestamp_oldest_dataset", $mints) if($hash->{LASTCMD} eq "minTimestamp"); ReadingsBulkUpdateValue ($hash, "timestamp_oldest_dataset", $mints) if($hash->{LASTCMD} eq "minTimestamp");
ReadingsBulkUpdateValue ($hash, "index_state", $idxstate) if($hash->{LASTCMD} ne "minTimestamp");
ReadingsBulkUpdateTimeState($hash,$brt,$rt,$state); ReadingsBulkUpdateTimeState($hash,$brt,$rt,$state);
readingsEndUpdate($hash, 1); readingsEndUpdate($hash, 1);
@ -1533,7 +1566,7 @@ return &$fret($hash,$opt,$prop);
} }
#################################################################################################### ####################################################################################################
# Abbruchroutine den ältesten Datensatz (Timestamp) in der DB bestimmen # Abbruchroutine DbRep_getInitData
#################################################################################################### ####################################################################################################
sub DbRep_getInitDataAborted(@) { sub DbRep_getInitDataAborted(@) {
my ($hash,$cause) = @_; my ($hash,$cause) = @_;
@ -1646,7 +1679,7 @@ sub DbRep_Main($$;$) {
Log3 ($name, 3, "DbRep $name - WARNING - old process $hash->{HELPER}{RUNNING_INDEX}{pid} will be killed now to start a new index operation"); Log3 ($name, 3, "DbRep $name - WARNING - old process $hash->{HELPER}{RUNNING_INDEX}{pid} will be killed now to start a new index operation");
BlockingKill($hash->{HELPER}{RUNNING_INDEX}); BlockingKill($hash->{HELPER}{RUNNING_INDEX});
} }
Log3 ($name, 4, "DbRep $name - Command: $opt $prop"); Log3 ($name, 3, "DbRep $name - Command: $opt $prop");
$hash->{HELPER}{RUNNING_INDEX} = BlockingCall("DbRep_Index", "$name|$prop", "DbRep_IndexDone", $to, "DbRep_IndexAborted", $hash); $hash->{HELPER}{RUNNING_INDEX} = BlockingCall("DbRep_Index", "$name|$prop", "DbRep_IndexDone", $to, "DbRep_IndexAborted", $hash);
ReadingsSingleUpdateValue ($hash, "state", "index operation in database is running - be patient and see Logfile !", 1); ReadingsSingleUpdateValue ($hash, "state", "index operation in database is running - be patient and see Logfile !", 1);
$hash->{HELPER}{RUNNING_INDEX}{loglevel} = 5 if($hash->{HELPER}{RUNNING_INDEX}); # Forum #77057 $hash->{HELPER}{RUNNING_INDEX}{loglevel} = 5 if($hash->{HELPER}{RUNNING_INDEX}); # Forum #77057
@ -6363,7 +6396,7 @@ sub DbRep_Index($) {
$li = $sql_idx; $li = $sql_idx;
$i++; $i++;
} }
Log3($name, 4, "DbRep $name - Index found in database:\n$ret"); Log3($name, 3, "DbRep $name - Index found in database:\n$ret");
$ret = "Index found in database:\n========================\n".$ret; $ret = "Index found in database:\n========================\n".$ret;
if($cmd !~ /list/) { if($cmd !~ /list/) {
@ -6372,6 +6405,7 @@ sub DbRep_Index($) {
if($sqldel) { if($sqldel) {
eval {@six = $dbh->selectrow_array($sqlava);}; eval {@six = $dbh->selectrow_array($sqlava);};
if (@six) { if (@six) {
Log3 ($name, 3, "DbRep $name - dropping index $idx ... ");
eval {$rows = $dbh->do($sqldel);}; eval {$rows = $dbh->do($sqldel);};
if ($@) { if ($@) {
if($cmd !~ /recreate/) { if($cmd !~ /recreate/) {

110
fhem/contrib/DS_Starter/93_DbRep.pm
View File

@ -58,6 +58,7 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch';
# Version History intern # Version History intern
our %DbRep_vNotesIntern = ( our %DbRep_vNotesIntern = (
"8.20.1" => "28.04.2019 set index verbose changed, check index \"Report_Idx\" in getInitData ",
"8.20.0" => "27.04.2019 don't save hash refs in central hash to prevent potential memory leak, new set \"index\" ". "8.20.0" => "27.04.2019 don't save hash refs in central hash to prevent potential memory leak, new set \"index\" ".
"command, \"repository\" added in Meta.json ", "command, \"repository\" added in Meta.json ",
"8.19.1" => "10.04.2019 adjust \$hash->{HELPER}{IDRETRIES} if value is negative ", "8.19.1" => "10.04.2019 adjust \$hash->{HELPER}{IDRETRIES} if value is negative ",
@ -1397,8 +1398,7 @@ return undef;
################################################################################### ###################################################################################
# First Init DB Connect # First Init DB Connect
# Verbindung zur DB aufbauen und den Timestamp des ältesten # Verbindung zur DB aufbauen und Datenbankeigenschaften ermitteln
# Datensatzes ermitteln
################################################################################### ###################################################################################
sub DbRep_firstconnect(@) { sub DbRep_firstconnect(@) {
my ($string) = @_; my ($string) = @_;
@ -1439,54 +1439,85 @@ return;
} }
#################################################################################################### ####################################################################################################
# den ältesten Datensatz (Timestamp) in der DB bestimmen # DatenDatenbankeigenschaften ermitteln
#################################################################################################### ####################################################################################################
sub DbRep_getInitData($) { sub DbRep_getInitData($) {
my ($string) = @_; my ($string) = @_;
my ($name,$opt,$prop,$fret) = split("\\|", $string); my ($name,$opt,$prop,$fret) = split("\\|", $string);
my $hash = $defs{$name}; my $hash = $defs{$name};
my $dbloghash = $hash->{dbloghash}; my $dbloghash = $hash->{dbloghash};
my $dbconn = $dbloghash->{dbconn}; my $dbconn = $dbloghash->{dbconn};
my $dbuser = $dbloghash->{dbuser}; my $dbuser = $dbloghash->{dbuser};
my $dblogname = $dbloghash->{NAME}; my $dblogname = $dbloghash->{NAME};
my $dbpassword = $attr{"sec$dblogname"}{secret}; my $dbmodel = $dbloghash->{MODEL};
my $mintsdef = "1970-01-01 01:00:00"; my $dbpassword = $attr{"sec$dblogname"}{secret};
my ($dbh,$sql,$err,$mints); my $mintsdef = "1970-01-01 01:00:00";
my $idxstate = "";
my ($dbh,$sql,$err,$mints);
# Background-Startzeit # Background-Startzeit
my $bst = [gettimeofday]; my $bst = [gettimeofday];
eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1, AutoInactiveDestroy => 1 }); }; eval { $dbh = DBI->connect("dbi:$dbconn", $dbuser, $dbpassword, { PrintError => 0, RaiseError => 1, AutoInactiveDestroy => 1 }); };
if ($@) { if ($@) {
$err = encode_base64($@,""); $err = encode_base64($@,"");
Log3 ($name, 2, "DbRep $name - $@"); Log3 ($name, 2, "DbRep $name - $@");
return "$name|''|''|$err"; return "$name|''|''|$err";
} }
# SQL-Startzeit # SQL-Startzeit
my $st = [gettimeofday]; my $st = [gettimeofday];
eval { $mints = $dbh->selectrow_array("SELECT min(TIMESTAMP) FROM history;"); }; # ältesten Datensatz der DB ermitteln
eval { $mints = $dbh->selectrow_array("SELECT min(TIMESTAMP) FROM history;"); };
$dbh->disconnect; # Report_Idx Status ermitteln
my ($ava,$sqlava);
my $idx = "Report_Idx";
if($dbmodel =~ /MYSQL/) {
$sqlava = "SHOW INDEX FROM history where Key_name='$idx';";
} elsif($dbmodel =~ /SQLITE/) {
$sqlava = "SELECT name FROM sqlite_master WHERE type='index' AND name='$idx';";
} elsif($dbmodel =~ /POSTGRESQL/) {
$sqlava = "SELECT indexname FROM pg_indexes WHERE tablename='history' and indexname ='$idx';";
}
eval {$ava = $dbh->selectrow_array($sqlava)};
if($@) {
$idxstate = "state of Index $idx can't be determined !";
Log3($name, 2, "DbRep $name - WARNING - $idxstate");
} else {
if($hash->{LASTCMD} ne "minTimestamp") {
if($ava) {
$idxstate = "Index $idx exists";
Log3($name, 3, "DbRep $name - $idxstate. Check ok");
} else {
$idxstate = "Index $idx doesn't exist. Please create the index by \"set $name index recreate_Report_Idx\" command !";
Log3($name, 2, "DbRep $name - WARNING - $idxstate");
}
}
}
# SQL-Laufzeit ermitteln $dbh->disconnect;
my $rt = tv_interval($st);
$mints = $mints?encode_base64($mints,""):encode_base64($mintsdef,""); # SQL-Laufzeit ermitteln
my $rt = tv_interval($st);
# Background-Laufzeit ermitteln $mints = $mints?encode_base64($mints,""):encode_base64($mintsdef,"");
my $brt = tv_interval($bst); $idxstate = encode_base64($idxstate,"");
# Background-Laufzeit ermitteln
my $brt = tv_interval($bst);
$rt = $rt.",".$brt; $rt = $rt.",".$brt;
no warnings 'uninitialized'; no warnings 'uninitialized';
Log3 ($name, 3, "DbRep $name - Initial data information retrieved successfully - total time used: $brt seconds"); Log3 ($name, 3, "DbRep $name - Initial data information retrieved successfully - total time used: ".sprintf("%.4f",$brt)." seconds");
return "$name|$mints|$rt|0|$opt|$prop|$fret"; return "$name|$mints|$rt|0|$opt|$prop|$fret|$idxstate";
} }
#################################################################################################### ####################################################################################################
# Auswertungsroutine den ältesten Datensatz (Timestamp) in der DB bestimmen # Auswertungsroutine DbRep_getInitData
#################################################################################################### ####################################################################################################
sub DbRep_getInitDataDone($) { sub DbRep_getInitDataDone($) {
my ($string) = @_; my ($string) = @_;
@ -1500,6 +1531,7 @@ sub DbRep_getInitDataDone($) {
my $opt = $a[4]; my $opt = $a[4];
my $prop = $a[5]; my $prop = $a[5];
my $fret = \&{$a[6]} if($a[6]); my $fret = \&{$a[6]} if($a[6]);
my $idxstate = decode_base64($a[7]);
my $dblogdevice = $hash->{HELPER}{DBLOGDEVICE}; my $dblogdevice = $hash->{HELPER}{DBLOGDEVICE};
$hash->{dbloghash} = $defs{$dblogdevice}; $hash->{dbloghash} = $defs{$dblogdevice};
my $dbconn = $hash->{dbloghash}{dbconn}; my $dbconn = $hash->{dbloghash}{dbconn};
@ -1518,6 +1550,7 @@ sub DbRep_getInitDataDone($) {
readingsBeginUpdate($hash); readingsBeginUpdate($hash);
ReadingsBulkUpdateValue ($hash, "timestamp_oldest_dataset", $mints) if($hash->{LASTCMD} eq "minTimestamp"); ReadingsBulkUpdateValue ($hash, "timestamp_oldest_dataset", $mints) if($hash->{LASTCMD} eq "minTimestamp");
ReadingsBulkUpdateValue ($hash, "index_state", $idxstate) if($hash->{LASTCMD} ne "minTimestamp");
ReadingsBulkUpdateTimeState($hash,$brt,$rt,$state); ReadingsBulkUpdateTimeState($hash,$brt,$rt,$state);
readingsEndUpdate($hash, 1); readingsEndUpdate($hash, 1);
@ -1533,7 +1566,7 @@ return &$fret($hash,$opt,$prop);
} }
#################################################################################################### ####################################################################################################
# Abbruchroutine den ältesten Datensatz (Timestamp) in der DB bestimmen # Abbruchroutine DbRep_getInitData
#################################################################################################### ####################################################################################################
sub DbRep_getInitDataAborted(@) { sub DbRep_getInitDataAborted(@) {
my ($hash,$cause) = @_; my ($hash,$cause) = @_;
@ -1646,7 +1679,7 @@ sub DbRep_Main($$;$) {
Log3 ($name, 3, "DbRep $name - WARNING - old process $hash->{HELPER}{RUNNING_INDEX}{pid} will be killed now to start a new index operation"); Log3 ($name, 3, "DbRep $name - WARNING - old process $hash->{HELPER}{RUNNING_INDEX}{pid} will be killed now to start a new index operation");
BlockingKill($hash->{HELPER}{RUNNING_INDEX}); BlockingKill($hash->{HELPER}{RUNNING_INDEX});
} }
Log3 ($name, 4, "DbRep $name - Command: $opt $prop"); Log3 ($name, 3, "DbRep $name - Command: $opt $prop");
$hash->{HELPER}{RUNNING_INDEX} = BlockingCall("DbRep_Index", "$name|$prop", "DbRep_IndexDone", $to, "DbRep_IndexAborted", $hash); $hash->{HELPER}{RUNNING_INDEX} = BlockingCall("DbRep_Index", "$name|$prop", "DbRep_IndexDone", $to, "DbRep_IndexAborted", $hash);
ReadingsSingleUpdateValue ($hash, "state", "index operation in database is running - be patient and see Logfile !", 1); ReadingsSingleUpdateValue ($hash, "state", "index operation in database is running - be patient and see Logfile !", 1);
$hash->{HELPER}{RUNNING_INDEX}{loglevel} = 5 if($hash->{HELPER}{RUNNING_INDEX}); # Forum #77057 $hash->{HELPER}{RUNNING_INDEX}{loglevel} = 5 if($hash->{HELPER}{RUNNING_INDEX}); # Forum #77057
@ -6363,7 +6396,7 @@ sub DbRep_Index($) {
$li = $sql_idx; $li = $sql_idx;
$i++; $i++;
} }
Log3($name, 4, "DbRep $name - Index found in database:\n$ret"); Log3($name, 3, "DbRep $name - Index found in database:\n$ret");
$ret = "Index found in database:\n========================\n".$ret; $ret = "Index found in database:\n========================\n".$ret;
if($cmd !~ /list/) { if($cmd !~ /list/) {
@ -6372,6 +6405,7 @@ sub DbRep_Index($) {
if($sqldel) { if($sqldel) {
eval {@six = $dbh->selectrow_array($sqlava);}; eval {@six = $dbh->selectrow_array($sqlava);};
if (@six) { if (@six) {
Log3 ($name, 3, "DbRep $name - dropping index $idx ... ");
eval {$rows = $dbh->do($sqldel);}; eval {$rows = $dbh->do($sqldel);};
if ($@) { if ($@) {
if($cmd !~ /recreate/) { if($cmd !~ /recreate/) {