FHEM/fhem-mirror
2
0
Fork 0
mirror of https://github.com/fhem/fhem-mirror.git synced 2025-04-08 01:14:19 +00:00
Code Issues Releases Wiki Activity

93_DbLog: contrib 5.6.2

Browse Source 
git-svn-id: https://svn.fhem.de/fhem/trunk@27105 2b470e98-0d58-463d-a4d8-8e2adae1ed80
This commit is contained in:
nasseeder1 2023-01-22 20:00:52 +00:00
parent 08153df21e
commit 87b3f005d6
1 changed files with 412 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

417
fhem/contrib/DS_Starter/93_DbLog.pm
View File

@ -1,5 +1,5 @@
############################################################################################################################################ ############################################################################################################################################
# $Id: 93_DbLog.pm 26923 2023-01-10 10:28:14Z DS_Starter $ # $Id: 93_DbLog.pm 27082 2023-01-18 22:08:25Z DS_Starter $
# #
# 93_DbLog.pm # 93_DbLog.pm
# written by Dr. Boris Neubert 2007-12-30 # written by Dr. Boris Neubert 2007-12-30
@ -8,7 +8,7 @@
# modified and maintained by Tobias Faust since 2012-06-26 until 2016 # modified and maintained by Tobias Faust since 2012-06-26 until 2016
# e-mail: tobias dot faust at online dot de # e-mail: tobias dot faust at online dot de
# #
# redesigned and maintained 2016-2023 by DS_Starter with credits by: JoeAllb, DeeSpe # redesigned and maintained 2016-2023 by DS_Starter
# e-mail: heiko dot maaz at t-online dot de # e-mail: heiko dot maaz at t-online dot de
# #
# reduceLog() created by Claudiu Schuster (rapster) adapted by DS_Starter # reduceLog() created by Claudiu Schuster (rapster) adapted by DS_Starter
@ -38,14 +38,17 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch';
# Version History intern by DS_Starter: # Version History intern by DS_Starter:
my %DbLog_vNotesIntern = ( my %DbLog_vNotesIntern = (
"5.5.12" => "10.01.2023 changed routine _DbLog_SBP_onRun_LogBulk ", "5.6.2" => "22.01.2023 check Syntax of DbLogValueFn attribute with Log output ",
"5.6.1" => "16.01.2023 rewrite sub _DbLog_SBP_connectDB, rewrite sub DbLog_ExecSQL, _DbLog_SBP_onRun_deleteOldDays ",
"5.6.0" => "11.01.2023 rename attribute 'bulkInsert' to 'insertMode' ",
"5.5.12" => "10.01.2023 changed routine _DbLog_SBP_onRun_LogSequential, edit CommandRef ",
"5.5.11" => "09.01.2023 more code rework / structured subroutines ", "5.5.11" => "09.01.2023 more code rework / structured subroutines ",
"5.5.10" => "07.01.2023 more code rework (_DbLog_SBP_checkDiscDelpars) and others, use dbh quote in _DbLog_SBP_onRun_LogBulk ". "5.5.10" => "07.01.2023 more code rework (_DbLog_SBP_checkDiscDelpars) and others, use dbh quote in _DbLog_SBP_onRun_LogSequential ".
"configCheck changed to use only one db connect + measuring the connection time, universal DBHU ", "configCheck changed to use only one db connect + measuring the connection time, universal DBHU ",
"5.5.9" => "28.12.2022 optimize \$hash->{HELPER}{TH}, \$hash->{HELPER}{TC}, mode in Define ". "5.5.9" => "28.12.2022 optimize \$hash->{HELPER}{TH}, \$hash->{HELPER}{TC}, mode in Define ".
"Forum: https://forum.fhem.de/index.php/topic,130588.msg1254073.html#msg1254073 ", "Forum: https://forum.fhem.de/index.php/topic,130588.msg1254073.html#msg1254073 ",
"5.5.8" => "27.12.2022 two-line output of long state messages, define LONGRUN_PID threshold ", "5.5.8" => "27.12.2022 two-line output of long state messages, define LONGRUN_PID threshold ",
"5.5.7" => "20.12.2022 cutted _DbLog_SBP_onRun_Log into _DbLog_SBP_onRun_LogArray and _DbLog_SBP_onRun_LogBulk ". "5.5.7" => "20.12.2022 cutted _DbLog_SBP_onRun_Log into _DbLog_SBP_onRun_LogArray and _DbLog_SBP_onRun_LogSequential ".
"__DbLog_SBP_onRun_LogCurrent, __DbLog_SBP_fieldArrays, some bugfixes, add drivers to configCheck, edit comref ", "__DbLog_SBP_onRun_LogCurrent, __DbLog_SBP_fieldArrays, some bugfixes, add drivers to configCheck, edit comref ",
"5.5.6" => "12.12.2022 Serialize with Storable instead of JSON, more code rework ", "5.5.6" => "12.12.2022 Serialize with Storable instead of JSON, more code rework ",
"5.5.5" => "11.12.2022 Array Log -> may be better error processing ", "5.5.5" => "11.12.2022 Array Log -> may be better error processing ",
@ -147,7 +150,6 @@ sub DbLog_Initialize {
$hash->{ShutdownFn} = "DbLog_Shutdown"; $hash->{ShutdownFn} = "DbLog_Shutdown";
$hash->{AttrList} = "addStateEvent:0,1 ". $hash->{AttrList} = "addStateEvent:0,1 ".
"asyncMode:1,0 ". "asyncMode:1,0 ".
"bulkInsert:1,0 ".
"commitMode:basic_ta:on,basic_ta:off,ac:on_ta:on,ac:on_ta:off,ac:off_ta:on ". "commitMode:basic_ta:on,basic_ta:off,ac:on_ta:on,ac:on_ta:off,ac:off_ta:on ".
"cacheEvents:2,1,0 ". "cacheEvents:2,1,0 ".
"cacheLimit ". "cacheLimit ".
@ -158,14 +160,13 @@ sub DbLog_Initialize {
"convertTimezone:UTC,none ". "convertTimezone:UTC,none ".
"DbLogSelectionMode:Exclude,Include,Exclude/Include ". "DbLogSelectionMode:Exclude,Include,Exclude/Include ".
"DbLogType:Current,History,Current/History,SampleFill/History ". "DbLogType:Current,History,Current/History,SampleFill/History ".
"SQLiteJournalMode:WAL,off ".
"SQLiteCacheSize ".
"dbSchema ". "dbSchema ".
"defaultMinInterval:textField-long ". "defaultMinInterval:textField-long ".
"disable:1,0 ". "disable:1,0 ".
"excludeDevs ". "excludeDevs ".
"expimpdir ". "expimpdir ".
"exportCacheAppend:1,0 ". "exportCacheAppend:1,0 ".
"insertMode:1,0 ".
"noSupportPK:1,0 ". "noSupportPK:1,0 ".
"noNotifyDev:1,0 ". "noNotifyDev:1,0 ".
"showproctime:1,0 ". "showproctime:1,0 ".
@ -174,6 +175,8 @@ sub DbLog_Initialize {
"syncEvents:1,0 ". "syncEvents:1,0 ".
"syncInterval ". "syncInterval ".
"showNotifyTime:1,0 ". "showNotifyTime:1,0 ".
"SQLiteJournalMode:WAL,off ".
"SQLiteCacheSize ".
"traceFlag:SQL,CON,ENC,DBD,TXN,ALL ". "traceFlag:SQL,CON,ENC,DBD,TXN,ALL ".
"traceLevel:0,1,2,3,4,5,6,7 ". "traceLevel:0,1,2,3,4,5,6,7 ".
"timeout ". "timeout ".
@ -190,6 +193,9 @@ sub DbLog_Initialize {
$hash->{SVG_sampleDataFn} = "DbLog_sampleDataFn"; $hash->{SVG_sampleDataFn} = "DbLog_sampleDataFn";
$hash->{prioSave} = 1; # Prio-Flag für save Reihenfolge, Forum: https://forum.fhem.de/index.php/topic,130588.msg1249277.html#msg1249277 $hash->{prioSave} = 1; # Prio-Flag für save Reihenfolge, Forum: https://forum.fhem.de/index.php/topic,130588.msg1249277.html#msg1249277
$hash->{AttrRenameMap} = { "bulkInsert" => "insertMode",
};
eval { FHEM::Meta::InitMod( __FILE__, $hash ) }; # für Meta.pm (https://forum.fhem.de/index.php/topic,97589.0.html) eval { FHEM::Meta::InitMod( __FILE__, $hash ) }; # für Meta.pm (https://forum.fhem.de/index.php/topic,97589.0.html)
return; return;
@ -404,22 +410,8 @@ sub DbLog_Attr {
return qq{"$aName" is not valid for database model "$hash->{MODEL}"}; return qq{"$aName" is not valid for database model "$hash->{MODEL}"};
} }
if( $aName eq 'valueFn' ) { if($aName =~ /[Vv]alueFn/) {
my %specials= ( my ($err, $func) = DbLog_checkSyntaxValueFn ($name, $aVal);
"%TIMESTAMP" => $name,
"%LASTTIMESTAMP" => $name,
"%DEVICE" => $name,
"%DEVICETYPE" => $name,
"%EVENT" => $name,
"%READING" => $name,
"%VALUE" => $name,
"%LASTVALUE" => $name,
"%UNIT" => $name,
"%IGNORE" => $name,
"%CN" => $name
);
my $err = perlSyntaxCheck($aVal, %specials);
return $err if($err); return $err if($err);
} }
@ -1236,19 +1228,11 @@ sub DbLog_Log {
my $clim = AttrVal ($name, 'cacheLimit', $dblog_cachedef); my $clim = AttrVal ($name, 'cacheLimit', $dblog_cachedef);
my $ce = AttrVal ($name, 'cacheEvents', 0); my $ce = AttrVal ($name, 'cacheEvents', 0);
if( $DbLogValueFn =~ m/^\s*(\{.*\})\s*$/s ) { # Funktion aus Device spezifischer DbLogValueFn validieren ($err, $DbLogValueFn) = DbLog_checkSyntaxValueFn ($name, $DbLogValueFn, $dev_name); # Funktion aus Device spezifischer DbLogValueFn validieren
$DbLogValueFn = $1; $DbLogValueFn = '' if($err);
}
else {
$DbLogValueFn = '';
}
if( $value_fn =~ m/^\s*(\{.*\})\s*$/s ) { # Funktion aus Attr valueFn validieren ($err, $value_fn) = DbLog_checkSyntaxValueFn ($name, $value_fn); # Funktion aus Attr valueFn validieren
$value_fn = $1; $value_fn = '' if($err);
}
else {
$value_fn = '';
}
eval { # one Transaction eval { # one Transaction
for (my $i = 0; $i < $max; $i++) { for (my $i = 0; $i < $max; $i++) {
@ -2206,10 +2190,10 @@ sub DbLog_SBP_onRun {
## Event Logging ## Event Logging
######################################################### #########################################################
if ($operation =~ /log_/xs) { if ($operation =~ /log_/xs) {
my $bi = $memc->{bi}; # Bulk-Insert 0|1 my $im = $memc->{im}; # Insert-Mode 0|1
if ($bi) { if ($im) {
_DbLog_SBP_onRun_LogBulk ( { subprocess => $subprocess, _DbLog_SBP_onRun_LogSequential ( { subprocess => $subprocess,
name => $name, name => $name,
memc => $memc, memc => $memc,
store => $store, store => $store,
@ -2469,6 +2453,10 @@ return $doNext;
# PrintError - handle attribute tells DBI to call the Perl warn( ) function # PrintError - handle attribute tells DBI to call the Perl warn( ) function
# (which typically results in errors being printed to the screen # (which typically results in errors being printed to the screen
# when encountered) # when encountered)
#
# For maximum reliability and for robustness against database corruption,
# SQLite should always be run with its default synchronous setting of FULL.
# https://sqlite.org/howtocorrupt.html
################################################################################### ###################################################################################
sub _DbLog_SBP_connectDB { sub _DbLog_SBP_connectDB {
my $paref = shift; my $paref = shift;
@ -2533,27 +2521,56 @@ sub _DbLog_SBP_connectDB {
if($utf8) { if($utf8) {
if($model eq "MYSQL") { if($model eq "MYSQL") {
$dbh->{mysql_enable_utf8} = 1; $dbh->{mysql_enable_utf8} = 1;
$dbh->do('set names "UTF8"'); ($err, undef) = _DbLog_SBP_dbhDo ($name, $dbh, 'set names "UTF8"');
return ($err, q{}) if($err);
} }
if($model eq "SQLITE") { if($model eq "SQLITE") {
$dbh->do('PRAGMA encoding="UTF-8"'); ($err, undef) = _DbLog_SBP_dbhDo ($name, $dbh, 'PRAGMA encoding="UTF-8"');
return ($err, q{}) if($err);
} }
} }
if ($model eq 'SQLITE') { if ($model eq 'SQLITE') {
$dbh->do("PRAGMA temp_store=MEMORY"); my @dos = ("PRAGMA temp_store=MEMORY",
$dbh->do("PRAGMA synchronous=FULL"); # For maximum reliability and for robustness against database corruption, "PRAGMA synchronous=FULL",
# SQLite should always be run with its default synchronous setting of FULL. "PRAGMA journal_mode=$sltjm",
# https://sqlite.org/howtocorrupt.html "PRAGMA cache_size=$sltcs"
);
$dbh->do("PRAGMA journal_mode=$sltjm"); for my $do (@dos) {
$dbh->do("PRAGMA cache_size=$sltcs"); ($err, undef) = _DbLog_SBP_dbhDo ($name, $dbh, $do);
return ($err, q{}) if($err);
}
} }
return ($err, $dbh); return ($err, $dbh);
} }
####################################################################################################
# einfaches Sdbh->do, return ERROR-String wenn Fehler bzw. die Anzahl der betroffenen Zeilen
####################################################################################################
sub _DbLog_SBP_dbhDo {
my $name = shift;
my $dbh = shift;
my $sql = shift;
my $info = shift // "simple do statement: $sql";
my $err = q{};
my $rv = q{};
Log3 ($name, 4, "DbLog $name - $info");
eval{ $rv = $dbh->do($sql);
1;
}
or do { $err = $@;
Log3 ($name, 2, "DbLog $name - ERROR - $@");
};
return ($err, $rv);
}
############################################################################ ############################################################################
# Datenbank Ping # Datenbank Ping
# ohne alarm (timeout) bleibt ping hängen wenn DB nicht # ohne alarm (timeout) bleibt ping hängen wenn DB nicht
@ -2592,11 +2609,37 @@ sub _DbLog_SBP_pingDB {
return $bool; return $bool;
} }
############################################################################
# DBH set
# PrintError = 1, RaiseError = 0
############################################################################
sub _DbLog_SBP_dbhPrintError {
my $dbh = shift;
$dbh->{PrintError} = 1;
$dbh->{RaiseError} = 0;
return;
}
############################################################################
# DBH set
# PrintError = 0, RaiseError = 1
############################################################################
sub _DbLog_SBP_dbhRaiseError {
my $dbh = shift;
$dbh->{PrintError} = 0;
$dbh->{RaiseError} = 1;
return;
}
################################################################# #################################################################
# SubProcess - Log-Routine # SubProcess - Log-Routine
# Bulk-Insert # Bulk-Insert
################################################################# #################################################################
sub _DbLog_SBP_onRun_LogBulk { sub _DbLog_SBP_onRun_LogSequential {
my $paref = shift; my $paref = shift;
my $subprocess = $paref->{subprocess}; my $subprocess = $paref->{subprocess};
@ -2715,9 +2758,8 @@ sub _DbLog_SBP_onRun_LogBulk {
$error = __DbLog_SBP_beginTransaction ($name, $dbh, $useta); $error = __DbLog_SBP_beginTransaction ($name, $dbh, $useta);
if(!$useta) { # generate errstr wenn keine TA if(!$useta) { # keine Transaktion: generate errstr, keine Ausnahme
$dbh->{PrintError} = 1; _DbLog_SBP_dbhPrintError ($dbh);
$dbh->{RaiseError} = 0;
} }
eval { for my $ds (@ins) { eval { for my $ds (@ins) {
@ -2728,7 +2770,6 @@ sub _DbLog_SBP_onRun_LogBulk {
Log3 ($name, 2, "DbLog $name - ERROR in >$operation< - ".$sth_ih->errstr); Log3 ($name, 2, "DbLog $name - ERROR in >$operation< - ".$sth_ih->errstr);
} }
else { else {
#$rv = 0 if($rv eq "0E0");
$ins_hist += $rv; $ins_hist += $rv;
} }
} }
@ -2747,6 +2788,7 @@ sub _DbLog_SBP_onRun_LogBulk {
Log3 ($name, 2, "DbLog $name - Transaction is switched off. Transferred data is lost."); Log3 ($name, 2, "DbLog $name - Transaction is switched off. Transferred data is lost.");
} }
_DbLog_SBP_dbhRaiseError ($dbh);
__DbLog_SBP_rollbackOnly ($name, $dbh, $history); __DbLog_SBP_rollbackOnly ($name, $dbh, $history);
$ret = { $ret = {
@ -2759,14 +2801,11 @@ sub _DbLog_SBP_onRun_LogBulk {
__DbLog_SBP_sendToParent ($subprocess, $ret); __DbLog_SBP_sendToParent ($subprocess, $ret);
$dbh->{PrintError} = 0;
$dbh->{RaiseError} = 1;
return; return;
}; };
$dbh->{PrintError} = 0; _DbLog_SBP_dbhRaiseError ($dbh);
$dbh->{RaiseError} = 1; __DbLog_SBP_commitOnly ($name, $dbh, $history);
if($ins_hist == $ceti) { if($ins_hist == $ceti) {
Log3 ($name, 4, "DbLog $name - $ins_hist of $ceti events inserted into table $history".($usepkh ? " using PK on columns $pkh" : "")); Log3 ($name, 4, "DbLog $name - $ins_hist of $ceti events inserted into table $history".($usepkh ? " using PK on columns $pkh" : ""));
@ -2779,8 +2818,6 @@ sub _DbLog_SBP_onRun_LogBulk {
Log3 ($name, 2, "DbLog $name - WARNING - only ".$ins_hist." of $ceti events inserted into table $history"); Log3 ($name, 2, "DbLog $name - WARNING - only ".$ins_hist." of $ceti events inserted into table $history");
} }
} }
__DbLog_SBP_commitOnly ($name, $dbh, $history);
} }
if ($operation eq 'importCachefile') { if ($operation eq 'importCachefile') {
@ -2954,11 +2991,14 @@ sub _DbLog_SBP_onRun_LogArray {
$error = __DbLog_SBP_beginTransaction ($name, $dbh, $useta); $error = __DbLog_SBP_beginTransaction ($name, $dbh, $useta);
eval { if(!$useta) { # keine Transaktion: generate errstr, keine Ausnahme
($tuples, $rows) = $sth_ih->execute_array( { ArrayTupleStatus => \@tuple_status } ); _DbLog_SBP_dbhPrintError ($dbh);
}; }
if ($@) { eval { ($tuples, $rows) = $sth_ih->execute_array( { ArrayTupleStatus => \@tuple_status } );
1;
}
or do {
$error = $@; $error = $@;
$nins_hist = $ceti; $nins_hist = $ceti;
@ -2966,18 +3006,31 @@ sub _DbLog_SBP_onRun_LogArray {
if($useta) { if($useta) {
$rowlback = $cdata; # nicht gespeicherte Datensätze nur zurück geben wenn Transaktion ein $rowlback = $cdata; # nicht gespeicherte Datensätze nur zurück geben wenn Transaktion ein
__DbLog_SBP_rollbackOnly ($name, $dbh, $history);
Log3 ($name, 4, "DbLog $name - Transaction is switched on. Transferred data is returned to the cache."); Log3 ($name, 4, "DbLog $name - Transaction is switched on. Transferred data is returned to the cache.");
} }
else { else {
__DbLog_SBP_commitOnly ($name, $dbh, $history);
Log3 ($name, 4, "DbLog $name - Transaction is switched off. Some or all of the transferred data will be lost. Note the following information."); Log3 ($name, 4, "DbLog $name - Transaction is switched off. Some or all of the transferred data will be lost. Note the following information.");
} }
}
else { _DbLog_SBP_dbhRaiseError ($dbh);
__DbLog_SBP_rollbackOnly ($name, $dbh, $history);
$ret = {
name => $name,
msg => $error,
ot => 0,
oper => $operation,
rowlback => $rowlback
};
__DbLog_SBP_sendToParent ($subprocess, $ret);
return;
};
_DbLog_SBP_dbhRaiseError ($dbh);
__DbLog_SBP_commitOnly ($name, $dbh, $history); __DbLog_SBP_commitOnly ($name, $dbh, $history);
}
no warnings 'uninitialized'; no warnings 'uninitialized';
@ -2987,7 +3040,7 @@ sub _DbLog_SBP_onRun_LogArray {
next if($status); # $status ist "1" wenn insert ok next if($status); # $status ist "1" wenn insert ok
Log3 ($name, 4, "DbLog $name - Insert into $history rejected".($usepkh ? " (possible PK violation) " : " ")."- TS: $timestamp[$tuple], Device: $device[$tuple], Reading: $reading[$tuple]"); Log3 ($name, 4, "DbLog $name - Insert into $history rejected".($usepkh ? " (possible PK violation) " : " ")."->\nTS: $timestamp[$tuple], Device: $device[$tuple], Reading: $reading[$tuple]");
$event[$tuple] =~ s/\|/_ESC_/gxs; # escape Pipe "|" $event[$tuple] =~ s/\|/_ESC_/gxs; # escape Pipe "|"
$reading[$tuple] =~ s/\|/_ESC_/gxs; $reading[$tuple] =~ s/\|/_ESC_/gxs;
@ -3273,7 +3326,7 @@ sub __DbLog_SBP_logLogmodes {
my $name = $paref->{name}; my $name = $paref->{name};
my $useta = $paref->{useta}; my $useta = $paref->{useta};
my $dbh = $store->{dbh}; my $dbh = $store->{dbh};
my $bi = $memc->{bi}; # Bulk-Insert 0|1 my $im = $memc->{im}; # Insert-Mode 0|1
my $DbLogType = $memc->{DbLogType}; # Log-Ziele my $DbLogType = $memc->{DbLogType}; # Log-Ziele
my $operation = $memc->{operation} // 'unknown'; # aktuell angeforderte Operation (log, etc.) my $operation = $memc->{operation} // 'unknown'; # aktuell angeforderte Operation (log, etc.)
@ -3283,7 +3336,7 @@ sub __DbLog_SBP_logLogmodes {
Log3 ($name, 4, "DbLog $name - Operation: $operation"); Log3 ($name, 4, "DbLog $name - Operation: $operation");
Log3 ($name, 5, "DbLog $name - DbLogType: $DbLogType"); Log3 ($name, 5, "DbLog $name - DbLogType: $DbLogType");
Log3 ($name, 4, "DbLog $name - AutoCommit: $ac, Transaction: $tm"); Log3 ($name, 4, "DbLog $name - AutoCommit: $ac, Transaction: $tm");
Log3 ($name, 4, "DbLog $name - Insert mode: ".($bi ? "Bulk" : "Array")); Log3 ($name, 4, "DbLog $name - Insert mode: ".($im ? "Sequential" : "Array"));
return; return;
} }
@ -3374,26 +3427,22 @@ sub _DbLog_SBP_onRun_deleteOldDays {
my $st = [gettimeofday]; # SQL-Startzeit my $st = [gettimeofday]; # SQL-Startzeit
if(defined ($cmd)) { if(defined ($cmd)) {
eval { $numdel = $dbh->do($cmd); (my $err, $numdel) = _DbLog_SBP_dbhDo ($name, $dbh, $cmd);
1;
}
or do { $error = $@;
Log3 ($name, 2, "DbLog $name - Error table $history - $error");
if ($err) {
$dbh->disconnect(); $dbh->disconnect();
delete $store->{dbh}; delete $store->{dbh};
$ret = { $ret = {
name => $name, name => $name,
msg => $error, msg => $err,
ot => 0, ot => 0,
oper => $operation oper => $operation
}; };
__DbLog_SBP_sendToParent ($subprocess, $ret); __DbLog_SBP_sendToParent ($subprocess, $ret);
return; return;
}; }
$numdel = 0 if($numdel == 0E0); $numdel = 0 if($numdel == 0E0);
$error = __DbLog_SBP_commitOnly ($name, $dbh, $history); $error = __DbLog_SBP_commitOnly ($name, $dbh, $history);
@ -3491,7 +3540,7 @@ return;
# $memc->{arguments} -> $infile # $memc->{arguments} -> $infile
# $memc->{operation} -> 'importCachefile' # $memc->{operation} -> 'importCachefile'
# $memc->{DbLogType} -> 'history' # $memc->{DbLogType} -> 'history'
# $memc->{bi} -> 0 # $memc->{im} -> 0
# #
################################################################# #################################################################
sub _DbLog_SBP_onRun_importCachefile { sub _DbLog_SBP_onRun_importCachefile {
@ -3546,7 +3595,7 @@ sub _DbLog_SBP_onRun_importCachefile {
Log3 ($name, 3, "DbLog $name - $msg"); Log3 ($name, 3, "DbLog $name - $msg");
$memc->{DbLogType} = 'history'; # nur history-Insert ! $memc->{DbLogType} = 'history'; # nur history-Insert !
$memc->{bi} = 0; # Array-Insert ! $memc->{im} = 0; # Array-Insert !
($error, $nins_hist, $rowlback) = _DbLog_SBP_onRun_LogArray ( { subprocess => $subprocess, ($error, $nins_hist, $rowlback) = _DbLog_SBP_onRun_LogArray ( { subprocess => $subprocess,
name => $name, name => $name,
@ -4186,32 +4235,6 @@ sub __DbLog_SBP_disconnectOnly {
return $err; return $err;
} }
#################################################################
# erstellt SQL für Insert Daten in die HISTORY! Tabelle
#################################################################
sub __DbLog_SBP_sqlInsHistory {
my $table = shift;
my $model = shift;
my $usepkh = shift;
my $sql;
if ($usepkh && $model eq 'MYSQL') {
$sql = "INSERT IGNORE INTO $table (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES ";
}
elsif ($usepkh && $model eq 'SQLITE') {
$sql = "INSERT OR IGNORE INTO $table (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES ";
}
elsif ($usepkh && $model eq 'POSTGRESQL') {
$sql = "INSERT INTO $table (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES ";
}
else { # ohne PK
$sql = "INSERT INTO $table (TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES ";
}
return $sql;
}
################################################################# #################################################################
# erstellt Statement Handle für Insert Daten in die # erstellt Statement Handle für Insert Daten in die
# angegebene Tabelle # angegebene Tabelle
@ -4453,7 +4476,7 @@ sub DbLog_SBP_sendLogData {
$memc->{nsupk} = AttrVal ($name, 'noSupportPK', 0); $memc->{nsupk} = AttrVal ($name, 'noSupportPK', 0);
$memc->{tl} = AttrVal ($name, 'traceLevel', 0); $memc->{tl} = AttrVal ($name, 'traceLevel', 0);
$memc->{tf} = AttrVal ($name, 'traceFlag', 'SQL'); $memc->{tf} = AttrVal ($name, 'traceFlag', 'SQL');
$memc->{bi} = AttrVal ($name, 'bulkInsert', 0); $memc->{im} = AttrVal ($name, 'insertMode', 0);
$memc->{verbose} = AttrVal ($name, 'verbose', 3); $memc->{verbose} = AttrVal ($name, 'verbose', 3);
$memc->{operation} = $oper; $memc->{operation} = $oper;
@ -4492,7 +4515,7 @@ sub DbLog_SBP_sendCommand {
$memc->{nsupk} = AttrVal ($name, 'noSupportPK', 0); $memc->{nsupk} = AttrVal ($name, 'noSupportPK', 0);
$memc->{tl} = AttrVal ($name, 'traceLevel', 0); $memc->{tl} = AttrVal ($name, 'traceLevel', 0);
$memc->{tf} = AttrVal ($name, 'traceFlag', 'SQL'); $memc->{tf} = AttrVal ($name, 'traceFlag', 'SQL');
$memc->{bi} = AttrVal ($name, 'bulkInsert', 0); $memc->{im} = AttrVal ($name, 'insertMode', 0);
$memc->{verbose} = AttrVal ($name, 'verbose', 3); $memc->{verbose} = AttrVal ($name, 'verbose', 3);
$memc->{operation} = $oper; $memc->{operation} = $oper;
$memc->{arguments} = $arg; $memc->{arguments} = $arg;
@ -5012,7 +5035,8 @@ sub DbLog_ExecSQL {
Log3 ($name, 4, "DbLog $name - Backdoor executing: $sql"); Log3 ($name, 4, "DbLog $name - Backdoor executing: $sql");
my $sth = DbLog_ExecSQL1($hash, $dbh, $sql); ($err, my $sth) = _DbLog_SBP_dbhDo ($name, $dbh, $sql);
$sth = 0 if($err);
__DbLog_SBP_commitOnly ($name, $dbh); __DbLog_SBP_commitOnly ($name, $dbh);
__DbLog_SBP_disconnectOnly ($name, $dbh); __DbLog_SBP_disconnectOnly ($name, $dbh);
@ -5020,24 +5044,6 @@ sub DbLog_ExecSQL {
return $sth; return $sth;
} }
sub DbLog_ExecSQL1 {
my $hash = shift;
my $dbh = shift;
my $sql = shift;
my $name = $hash->{NAME};
my $sth;
eval { $sth = $dbh->do($sql); };
if($@) {
Log3 ($name, 2, "DbLog $name - ERROR: $@");
return 0;
}
return $sth;
}
################################################################ ################################################################
# #
# GET Funktion # GET Funktion
@ -5944,25 +5950,25 @@ sub DbLog_configcheck {
### Check Betriebsmodus ### Check Betriebsmodus
####################################################################### #######################################################################
my $mode = $hash->{MODE}; my $mode = $hash->{MODE};
my $bi = AttrVal($name, 'bulkInsert', 0); my $im = AttrVal($name, 'insertMode', 0);
my $sfx = AttrVal("global", 'language', 'EN'); my $sfx = AttrVal("global", 'language', 'EN');
$sfx = $sfx eq "EN" ? "" : "_$sfx"; $sfx = $sfx eq "EN" ? "" : "_$sfx";
$check .= "<u><b>Result of insert mode check</u></b><br><br>"; $check .= "<u><b>Result of insert mode check</u></b><br><br>";
if (!$bi) { if (!$im) {
$bi = "Array"; $im = "Array";
$check .= "Insert mode of DbLog-device $name is: $bi <br>"; $check .= "Insert mode of DbLog-device $name is: $im <br>";
$check .= "Rating: ".$ok."<br>";
$rec = "Setting attribute \"bulkInsert\" to \"1\" may result a higher write performance in most cases. ";
$rec .= "Feel free to try this mode.";
}
else {
$bi = "Bulk";
$check .= "Insert mode of DbLog-device $name is: $bi <br>";
$check .= "Rating: ".$ok."<br>"; $check .= "Rating: ".$ok."<br>";
$rec = "settings o.k."; $rec = "settings o.k.";
} }
else {
$im = "Sequential";
$check .= "Insert mode of DbLog-device $name is: $im <br>";
$check .= "Rating: ".$ok."<br>";
$rec = qq(Setting attribute "insertMode" to "0" (or delete it) may result a higher write performance in most cases. );
$rec .= "Feel free to try this mode.";
}
$check .= "<b>Recommendation:</b> $rec <br><br>"; $check .= "<b>Recommendation:</b> $rec <br><br>";
### Check Plot Erstellungsmodus ### Check Plot Erstellungsmodus
@ -7054,6 +7060,44 @@ sub DbLog_checkUsePK {
return ($upkh,$upkc,$pkh,$pkc); return ($upkh,$upkc,$pkh,$pkc);
} }
################################################################
# Syntaxcheck von Attr valueFn und DbLogValueFn
# Rückgabe von Error oder der gesäuberten Funktion
################################################################
sub DbLog_checkSyntaxValueFn {
my $name = shift;
my $func = shift;
my $devname = shift // q{};
my $err = q{};
if ($func !~ m/^\s*(\{.*\})\s*$/s) {
return "Error while syntax checking. The function has to be enclosed by curly brackets.";
}
my %specials= (
"%TIMESTAMP" => $name,
"%LASTTIMESTAMP" => $name,
"%DEVICE" => $name,
"%DEVICETYPE" => $name,
"%EVENT" => $name,
"%READING" => $name,
"%VALUE" => $name,
"%LASTVALUE" => $name,
"%UNIT" => $name,
"%IGNORE" => $name,
"%CN" => $name
);
$err = perlSyntaxCheck ($func, %specials);
Log3 ($name, 1, "DbLog $name - Syntaxcheck <$devname> attribute DbLogValueFn: \n".$err) if($err && $devname);
$func =~ s/^\s*(\{.*\})\s*$/$1/s;
return ($err, $func);
}
################################################################ ################################################################
# Routine für FHEMWEB Detailanzeige # Routine für FHEMWEB Detailanzeige
################################################################ ################################################################
@ -7551,13 +7595,13 @@ sub DbLog_setVersionInfo {
if($modules{$type}{META}{x_prereqs_src} && !$hash->{HELPER}{MODMETAABSENT}) { # META-Daten sind vorhanden if($modules{$type}{META}{x_prereqs_src} && !$hash->{HELPER}{MODMETAABSENT}) { # META-Daten sind vorhanden
$modules{$type}{META}{version} = "v".$v; # Version aus META.json überschreiben, Anzeige mit {Dumper $modules{DbLog}{META}} $modules{$type}{META}{version} = "v".$v; # Version aus META.json überschreiben, Anzeige mit {Dumper $modules{DbLog}{META}}
if($modules{$type}{META}{x_version}) { # {x_version} ( nur gesetzt wenn $Id: 93_DbLog.pm 26923 2022-12-29 10:28:14Z DS_Starter $ im Kopf komplett! vorhanden ) if($modules{$type}{META}{x_version}) { # {x_version} ( nur gesetzt wenn $Id: 93_DbLog.pm 27082 2023-01-18 22:08:25Z DS_Starter $ im Kopf komplett! vorhanden )
$modules{$type}{META}{x_version} =~ s/1\.1\.1/$v/xsg; $modules{$type}{META}{x_version} =~ s/1\.1\.1/$v/xsg;
} }
else { else {
$modules{$type}{META}{x_version} = $v; $modules{$type}{META}{x_version} = $v;
} }
return $@ unless (FHEM::Meta::SetInternals($hash)); # FVERSION wird gesetzt ( nur gesetzt wenn $Id: 93_DbLog.pm 26923 2022-12-29 10:28:14Z DS_Starter $ im Kopf komplett! vorhanden ) return $@ unless (FHEM::Meta::SetInternals($hash)); # FVERSION wird gesetzt ( nur gesetzt wenn $Id: 93_DbLog.pm 27082 2023-01-18 22:08:25Z DS_Starter $ im Kopf komplett! vorhanden )
if(__PACKAGE__ eq "FHEM::$type" || __PACKAGE__ eq $type) { if(__PACKAGE__ eq "FHEM::$type" || __PACKAGE__ eq $type) {
# es wird mit Packages gearbeitet -> Perl übliche Modulversion setzen # es wird mit Packages gearbeitet -> Perl übliche Modulversion setzen
# mit {<Modul>->VERSION()} im FHEMWEB kann Modulversion abgefragt werden # mit {<Modul>->VERSION()} im FHEMWEB kann Modulversion abgefragt werden
@ -8357,8 +8401,7 @@ return;
<tr><td> </td><td>In principle, the data is immediately available in the database. </td></tr> <tr><td> </td><td>In principle, the data is immediately available in the database. </td></tr>
<tr><td> </td><td>Very little to no data is lost when FHEM crashes. </td></tr> <tr><td> </td><td>Very little to no data is lost when FHEM crashes. </td></tr>
<tr><td> </td><td><b>Disadvantages:</b> </td></tr> <tr><td> </td><td><b>Disadvantages:</b> </td></tr>
<tr><td> </td><td>The data is only short cached and will be lost if the database is unavailable or malfunctions. </td></tr> <tr><td> </td><td>An alternative storage in the file system (in case of database problems) is not supported. </td></tr>
<tr><td> </td><td>Alternative storage in the file system is not supported. </td></tr>
<tr><td> </td><td> </td></tr> <tr><td> </td><td> </td></tr>
<tr><td> 1 - </td><td><b>Asynchroner Log-Modus.</b> The data to be logged is first cached in a memory cache and written to the database </td></tr> <tr><td> 1 - </td><td><b>Asynchroner Log-Modus.</b> The data to be logged is first cached in a memory cache and written to the database </td></tr>
<tr><td> </td><td>depending on a <a href="#DbLog-attr-syncInterval">time interval</a> or <a href="#DbLog-attr-cacheLimit">fill level</a> of the cache. </td></tr> <tr><td> </td><td>depending on a <a href="#DbLog-attr-syncInterval">time interval</a> or <a href="#DbLog-attr-cacheLimit">fill level</a> of the cache. </td></tr>
@ -8376,25 +8419,6 @@ return;
</ul> </ul>
<br> <br>
<ul>
<a id="DbLog-attr-bulkInsert"></a>
<li><b>bulkInsert</b>
<ul>
<code>attr &lt;device&gt; bulkInsert [1|0] </code><br><br>
Toggles the insert mode between "Array" and "Bulk". <br>
The bulk mode leads to a considerable performance increase when inserting a large number of data records into the
history table, especially in asynchronous mode.
To get the full performance increase, in this case the attribute "DbLogType" should <b>not</b> contain the
current table. <br>
In contrast, the "Array" mode has the advantage over "Bulk" that faulty data records are extracted and reported in the
log file. <br>
(default: 0=Array)
</ul>
</ul>
</li>
<br>
<ul> <ul>
<a id="DbLog-attr-commitMode"></a> <a id="DbLog-attr-commitMode"></a>
<li><b>commitMode</b> <li><b>commitMode</b>
@ -8872,6 +8896,30 @@ attr SMA_Energymeter DbLogValueFn
</ul> </ul>
<br> <br>
<ul>
<a id="DbLog-attr-insertMode"></a>
<li><b>insertMode</b>
<ul>
<code>attr &lt;device&gt; insertMode [1|0] </code><br><br>
Toggles the insert mode of the database interface. <br><br>
<ul>
<table>
<colgroup> <col width=5%> <col width=95%> </colgroup>
<tr><td> 0 - </td><td>The data is passed as an array to the database interface. </td></tr>
<tr><td> </td><td>It is in most cases the most performant way to insert a lot of data into the database at once. </td></tr>
<tr><td> 1 - </td><td>The records are passed sequentially to the database interface and inserted into the DB. </td></tr>
</table>
</ul>
<br>
(default: 0)
</ul>
</ul>
</li>
<br>
<ul> <ul>
<a id="DbLog-attr-noNotifyDev"></a> <a id="DbLog-attr-noNotifyDev"></a>
<li><b>noNotifyDev</b> <li><b>noNotifyDev</b>
@ -10018,8 +10066,7 @@ attr SMA_Energymeter DbLogValueFn
<tr><td> </td><td>Die Daten stehen im Prinzip sofort in der Datenbank zur Verfügung. </td></tr> <tr><td> </td><td>Die Daten stehen im Prinzip sofort in der Datenbank zur Verfügung. </td></tr>
<tr><td> </td><td>Bei einem Absturz von FHEM gehen sehr wenige bis keine Daten verloren. </td></tr> <tr><td> </td><td>Bei einem Absturz von FHEM gehen sehr wenige bis keine Daten verloren. </td></tr>
<tr><td> </td><td><b>Nachteile:</b> </td></tr> <tr><td> </td><td><b>Nachteile:</b> </td></tr>
<tr><td> </td><td>Die Daten werden nur kurz zwischengespeichert und gehen verloren wenn die Datenbank nicht verfügbar ist </td></tr> <tr><td> </td><td>Eine alternative Speicherung im Filesystem (bei Datenbankproblemen) wird nicht unterstützt. </td></tr>
<tr><td> </td><td>oder fehlerhaft arbeitet. Eine alternative Speicherung im Filesystem wird nicht unterstützt. </td></tr>
<tr><td> </td><td> </td></tr> <tr><td> </td><td> </td></tr>
<tr><td> </td><td> </td></tr> <tr><td> </td><td> </td></tr>
<tr><td> 1 - </td><td><b>Asynchroner Log-Modus.</b> Die zu loggenden Daten werden zunächst in einem Memory Cache zwischengespeichert </td></tr> <tr><td> 1 - </td><td><b>Asynchroner Log-Modus.</b> Die zu loggenden Daten werden zunächst in einem Memory Cache zwischengespeichert </td></tr>
@ -10038,24 +10085,6 @@ attr SMA_Energymeter DbLogValueFn
</ul> </ul>
<br> <br>
<ul>
<a id="DbLog-attr-bulkInsert"></a>
<li><b>bulkInsert</b>
<ul>
<code>attr &lt;device&gt; bulkInsert [1|0] </code><br><br>
Schaltet den Insert-Modus zwischen "Array" und "Bulk" um. <br>
Der Bulk Modus führt beim Insert von sehr vielen Datensätzen in die history-Tabelle zu einer erheblichen
Performancesteigerung vor allem im asynchronen Mode. Um die volle Performancesteigerung zu erhalten, sollte in
diesem Fall das Attribut "DbLogType" <b>nicht</b> die current-Tabelle enthalten. <br>
Demgegenüber hat der Modus "Array" gegenüber "Bulk" den Vorteil, dass fehlerhafte Datensätze extrahiert und im
Logfile reported werden. <br>
(default: 0=Array)
</ul>
</ul>
</li>
<br>
<ul> <ul>
<a id="DbLog-attr-cacheEvents"></a> <a id="DbLog-attr-cacheEvents"></a>
<li><b>cacheEvents</b> <li><b>cacheEvents</b>
@ -10553,6 +10582,30 @@ attr SMA_Energymeter DbLogValueFn
</ul> </ul>
<br> <br>
<ul>
<a id="DbLog-attr-insertMode"></a>
<li><b>insertMode</b>
<ul>
<code>attr &lt;device&gt; insertMode [1|0] </code><br><br>
Schaltet den Insert-Modus der Datenbankschnittstelle um. <br><br>
<ul>
<table>
<colgroup> <col width=5%> <col width=95%> </colgroup>
<tr><td> 0 - </td><td>Die Daten werden als Array der Datenbankschnittstelle übergeben. </td></tr>
<tr><td> </td><td>Es ist in den meisten Fällen der performanteste Weg viele Daten auf einmal in die Datenbank einzufügen. </td></tr>
<tr><td> 1 - </td><td>Die Datensätze werden sequentiell der Datenbankschnittstelle übergeben und in die DB eingefügt. </td></tr>
</table>
</ul>
<br>
(default: 0)
</ul>
</ul>
</li>
<br>
<ul> <ul>
<a id="DbLog-attr-noNotifyDev"></a> <a id="DbLog-attr-noNotifyDev"></a>
<li><b>noNotifyDev</b> <li><b>noNotifyDev</b>