mirror of
https://github.com/fhem/fhem-mirror.git
synced 2025-01-31 06:39:11 +00:00
93_DbLog: Support of utf8mb4 collation, minor fixes
git-svn-id: https://svn.fhem.de/fhem/trunk@27243 2b470e98-0d58-463d-a4d8-8e2adae1ed80
This commit is contained in:
nasseeder1
parent
de901959c8
commit
91cab10ff8
@ -1,5 +1,6 @@
|
||||
# Add changes at the top of the list. Keep it in ASCII, and 80-char wide.
|
||||
# Do not insert empty lines here, update check depends on it.
|
||||
- feature: 93_DbLog: Support of utf8mb4 collation, minor fixes
|
||||
- feature: 49_SSCam: Telegram send attributes: new key option => silent,
|
||||
peer can be fetched from r:<reading>
|
||||
- feature: 72_FRITZBOX: Anzeigen aus luaInfo verschönert
|
||||
|
||||
@ -38,6 +38,9 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch';
|
||||
|
||||
# Version History intern by DS_Starter:
|
||||
my %DbLog_vNotesIntern = (
|
||||
"5.8.1" => "13.02.2023 change field type of DbLogInclude, DbLogExclude to textField-long, configCheck evaluate collation ".
|
||||
"_DbLog_SBP_connectDB: UTF8 -> evaluate DB character/collation set and use it for ".
|
||||
"setting names connection collation ",
|
||||
"5.8.0" => "30.01.2023 new Get menu for a selection of getters, fix creation of new subprocess during shutdown sequence ",
|
||||
"5.7.0" => "25.01.2023 send Log3() data back ro parent process, improve _DbLog_dbReadings function ",
|
||||
"5.6.2" => "22.01.2023 check Syntax of DbLogValueFn attribute with Log output, Forum:#131777 ",
|
||||
@ -201,8 +204,8 @@ sub DbLog_Initialize {
|
||||
"verbose4Devs ".
|
||||
$readingFnAttributes;
|
||||
|
||||
addToAttrList("DbLogInclude");
|
||||
addToAttrList("DbLogExclude");
|
||||
addToAttrList("DbLogInclude:textField-long");
|
||||
addToAttrList("DbLogExclude:textField-long");
|
||||
addToAttrList("DbLogValueFn:textField-long");
|
||||
|
||||
$hash->{FW_detailFn} = "DbLog_fhemwebFn";
|
||||
@ -1339,7 +1342,7 @@ sub DbLog_Log {
|
||||
$DoIt = 1 if($DbLogSelectionMode =~ m/Exclude/ );
|
||||
|
||||
if($DbLogExclude && $DbLogSelectionMode =~ m/Exclude/) { # Bsp: "(temperature|humidity):300,battery:3600:force"
|
||||
my @v1 = split(/,/, $DbLogExclude);
|
||||
my @v1 = DbLog_attrLong2Array ($DbLogExclude, ',');
|
||||
|
||||
for (my $i = 0; $i < int(@v1); $i++) {
|
||||
my @v2 = split /:/, $v1[$i];
|
||||
@ -1363,7 +1366,7 @@ sub DbLog_Log {
|
||||
# Im Endeffekt genau die gleiche Pruefung, wie fuer DBLogExclude, lediglich mit umgegkehrtem Ergebnis.
|
||||
if($DoIt == 0) {
|
||||
if($DbLogInclude && ($DbLogSelectionMode =~ m/Include/)) {
|
||||
my @v1 = split /,/, $DbLogInclude;
|
||||
my @v1 = DbLog_attrLong2Array ($DbLogInclude, ',');
|
||||
|
||||
for (my $i = 0; $i < int(@v1); $i++) {
|
||||
my @v2 = split /:/, $v1[$i];
|
||||
@ -1851,20 +1854,24 @@ sub _DbLog_checkDefMinInt {
|
||||
my $force;
|
||||
my $DoIt = 1;
|
||||
|
||||
my $defminint = AttrVal($name, "defaultMinInterval", undef);
|
||||
my $defminint = AttrVal ($name, "defaultMinInterval", undef);
|
||||
return $DoIt if(!$defminint); # Attribut "defaultMinInterval" nicht im DbLog gesetzt -> kein ToDo
|
||||
|
||||
my $DbLogExclude = AttrVal ($dev_name, "DbLogExclude", undef);
|
||||
$DbLogExclude = join ",", DbLog_attrLong2Array ($DbLogExclude, ',');
|
||||
|
||||
my $DbLogInclude = AttrVal ($dev_name, "DbLogInclude", undef);
|
||||
$DbLogInclude = join ",", DbLog_attrLong2Array ($DbLogInclude, ',');
|
||||
|
||||
$defminint =~ s/[\s\n]/,/g;
|
||||
my @adef = split(/,/, $defminint);
|
||||
my @adef = split /,/, $defminint;
|
||||
my $inex = ($DbLogExclude ? $DbLogExclude."," : "").($DbLogInclude ? $DbLogInclude : "");
|
||||
|
||||
if($inex) { # Quelldevice hat DbLogExclude und/oder DbLogInclude gesetzt
|
||||
my @ie = split(/,/, $inex);
|
||||
my @ie = split /,/, $inex;
|
||||
|
||||
for (my $k = 0; $k < int(@ie); $k++) { # Bsp. für das auszuwertende Element
|
||||
my @rif = split(/:/, $ie[$k]); # "(temperature|humidity):300:force"
|
||||
my @rif = split /:/, $ie[$k]; # "(temperature|humidity):300:force"
|
||||
|
||||
if($reading =~ m,^$rif[0]$, && $rif[1]) { # aktuelles Reading matcht auf Regexp und minInterval ist angegeben
|
||||
return $DoIt; # Reading wurde bereits geprüft -> kein Überschreiben durch $defminint
|
||||
@ -1873,8 +1880,8 @@ sub _DbLog_checkDefMinInt {
|
||||
}
|
||||
|
||||
for (my $l = 0; $l < int(@adef); $l++) {
|
||||
my @adefelem = split("::", $adef[$l]); # Bsp. für ein defaulMInInterval Element:
|
||||
my @dvs = devspec2array($adefelem[0]); # device::interval[::force]
|
||||
my @adefelem = split "::", $adef[$l]; # Bsp. für ein defaulMInInterval Element:
|
||||
my @dvs = devspec2array ($adefelem[0]); # device::interval[::force]
|
||||
|
||||
if(@dvs) {
|
||||
for (@dvs) {
|
||||
@ -2597,8 +2604,27 @@ sub _DbLog_SBP_connectDB {
|
||||
if($utf8) {
|
||||
if($model eq "MYSQL") {
|
||||
$dbh->{mysql_enable_utf8} = 1;
|
||||
($err, undef) = _DbLog_SBP_dbhDo ($name, $dbh, 'set names "UTF8"', $subprocess);
|
||||
|
||||
($err, my @se) = _DbLog_prepExecQueryOnly ($name, $dbh, "SHOW VARIABLES LIKE 'collation_database'");
|
||||
return ($err, q{}) if($err);
|
||||
|
||||
my $dbcharset = @se ? $se[1] : 'noresult';
|
||||
|
||||
_DbLog_SBP_Log3Parent ( { name => $name,
|
||||
level => 4,
|
||||
msg => qq(Database Character set is >$dbcharset<),
|
||||
oper => 'log3parent',
|
||||
subprocess => $subprocess
|
||||
}
|
||||
);
|
||||
|
||||
if ($dbcharset !~ /noresult|ucs2|utf16|utf32/ixs) { # Impermissible Client Character Sets -> https://dev.mysql.com/doc/refman/8.0/en/charset-connection.html
|
||||
my $collation = $dbcharset;
|
||||
$dbcharset = (split '_', $collation, 2)[0];
|
||||
|
||||
($err, undef) = _DbLog_SBP_dbhDo ($name, $dbh, qq(set names "$dbcharset" collate "$collation"), $subprocess); # set names utf8 collate utf8_general_ci
|
||||
return ($err, q{}) if($err);
|
||||
}
|
||||
}
|
||||
|
||||
if($model eq "SQLITE") {
|
||||
@ -7073,7 +7099,7 @@ sub DbLog_configcheck {
|
||||
"Rating: ".$nok."<br>";
|
||||
$check .= "<br>";
|
||||
|
||||
### Connection und Encoding check
|
||||
### Connection und Collation check
|
||||
#######################################################################
|
||||
my $st = [gettimeofday]; # Startzeit
|
||||
my $dbh = _DbLog_getNewDBHandle ($hash) || return "Can't connect to database.";
|
||||
@ -7085,20 +7111,24 @@ sub DbLog_configcheck {
|
||||
my ($chutf8mod,$chutf8dat);
|
||||
|
||||
if ($dbmodel =~ /MYSQL/) {
|
||||
($err, @ce) = _DbLog_prepExecQueryOnly ($name, $dbh, "SHOW VARIABLES LIKE 'character_set_connection'");
|
||||
($err, @ce) = _DbLog_prepExecQueryOnly ($name, $dbh, qq(SHOW VARIABLES LIKE 'collation_connection')); # character_set_connection
|
||||
$chutf8mod = @ce ? uc($ce[1]) : "no result";
|
||||
|
||||
($err, @se) = _DbLog_prepExecQueryOnly ($name, $dbh, "SHOW VARIABLES LIKE 'character_set_database'");
|
||||
($err, @se) = _DbLog_prepExecQueryOnly ($name, $dbh, qq(SHOW VARIABLES LIKE 'collation_database')); # character_set_database
|
||||
$chutf8dat = @se ? uc($se[1]) : "no result";
|
||||
|
||||
if($chutf8mod eq $chutf8dat) {
|
||||
if ($chutf8dat =~ /utf8mb4/xsi && $chutf8mod eq $chutf8dat) {
|
||||
$rec = "settings o.k.";
|
||||
}
|
||||
elsif ($chutf8dat !~ /utf8mb4/xsi && $chutf8mod eq $chutf8dat) {
|
||||
$rec = "The collation of the database should be changed to 'utf8mb4_bin' so that umlauts and all special characters can be stored. <br>";
|
||||
$rec .= "You can easy do that with the DbRep command <b>set <DbRep-Device> migrateCollation utf8mb4_bin</b>. <br>";
|
||||
}
|
||||
else {
|
||||
$rec = "Both encodings should be identical. You can adjust the usage of UTF8 connection by setting the UTF8 parameter in file '$hash->{CONFIGURATION}' to the right value. ";
|
||||
}
|
||||
|
||||
if(uc($chutf8mod) ne "UTF8" && uc($chutf8dat) ne "UTF8") {
|
||||
if ($chutf8mod !~ /utf8/xsi) {
|
||||
$dbdhint = "";
|
||||
}
|
||||
else {
|
||||
@ -7162,9 +7192,9 @@ sub DbLog_configcheck {
|
||||
return $check;
|
||||
}
|
||||
|
||||
$check .= "<u><b>Result of encoding check</u></b><br><br>";
|
||||
$check .= "Encoding used by Client (connection): $chutf8mod <br>" if($dbmodel !~ /SQLITE/);
|
||||
$check .= "Encoding used by DB $dbname: $chutf8dat <br>";
|
||||
$check .= "<u><b>Result of collation check</u></b><br><br>";
|
||||
$check .= "Collation used by Client (connection): $chutf8mod <br>" if($dbmodel !~ /SQLITE/);
|
||||
$check .= "Collation used by DB $dbname: $chutf8dat <br>";
|
||||
$check .= $dbmodel =~ /SQLITE/ ? "Rating: ".$ok."<br>" :
|
||||
$rec =~ /settings\so.k./xs ? "Rating: ".$ok."<br>" :
|
||||
"Rating: ".$warn."<br>";
|
||||
@ -7452,7 +7482,7 @@ sub DbLog_configcheck {
|
||||
|
||||
if (!@six) {
|
||||
$check .= "The index 'Search_Idx' is missing. <br>";
|
||||
$rec = "You can create the index by executing statement <b>'CREATE INDEX Search_Idx ON `$history` (DEVICE, READING, TIMESTAMP) USING BTREE;'</b> <br>";
|
||||
$rec = "You can create the index by the DbRep command <b>set <DbRep-Device> index recreate_search_Idx</b> <br>";
|
||||
$rec .= "Depending on your database size this command may running a long time. <br>";
|
||||
$rec .= "Please make sure the device '$name' is operating in asynchronous mode to avoid FHEM from blocking when creating the index. <br>";
|
||||
$rec .= "<b>Note:</b> If you have just created another index which covers the same fields and order as suggested (e.g. a primary key) you don't need to create the 'Search_Idx' as well ! <br>";
|
||||
@ -7473,6 +7503,7 @@ sub DbLog_configcheck {
|
||||
$rec = "The index should contain the fields 'DEVICE', 'TIMESTAMP', 'READING'. ";
|
||||
$rec .= "You can change the index by executing e.g. <br>";
|
||||
$rec .= "<b>'ALTER TABLE `$history` DROP INDEX `Search_Idx`, ADD INDEX `Search_Idx` (`DEVICE`, `READING`, `TIMESTAMP`) USING BTREE;'</b> <br>";
|
||||
$rec .= "The DbRep command <b>set <DbRep-Device> index recreate_search_Idx</b> is doing the same for you. <br>";
|
||||
$rec .= "Depending on your database size this command may running a long time. <br>";
|
||||
}
|
||||
}
|
||||
@ -7838,19 +7869,19 @@ sub DbLog_AddLog {
|
||||
$found = 1 if($rd =~ m/^$rdspec$/); # Reading gefunden
|
||||
|
||||
if($DbLogExclude && !$nce) {
|
||||
my @v1 = split(/,/, $DbLogExclude);
|
||||
my @v1 = DbLog_attrLong2Array ($DbLogExclude, ',');
|
||||
|
||||
for (my $i = 0; $i < int(@v1); $i++) {
|
||||
my @v2 = split(/:/, $v1[$i]); # MinInterval wegschneiden, Bsp: "(temperature|humidity):600,battery:3600"
|
||||
my @v2 = split /:/, $v1[$i]; # MinInterval wegschneiden, Bsp: "(temperature|humidity):600,battery:3600"
|
||||
|
||||
if($rd =~ m,^$v2[0]$,) { # Reading matcht $DbLogExclude -> ausschließen vom addLog
|
||||
$do = 0;
|
||||
|
||||
if($DbLogInclude) {
|
||||
my @v3 = split(/,/, $DbLogInclude);
|
||||
my @v3 = DbLog_attrLong2Array ($DbLogInclude, ',');
|
||||
|
||||
for (my $i = 0; $i < int(@v3); $i++) {
|
||||
my @v4 = split(/:/, $v3[$i]);
|
||||
my @v4 = split /:/, $v3[$i];
|
||||
$do = 1 if($rd =~ m,^$v4[0]$,); # Reading matcht $DbLogInclude -> wieder in addLog einschließen
|
||||
}
|
||||
}
|
||||
@ -8180,6 +8211,22 @@ sub DbLog_charfilter {
|
||||
return($txt);
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Einen Attributinhalt vom Typ textField-long splitten und als
|
||||
# Array zurückgeben
|
||||
# Optional kann das Split-Zeichen, default ',', angegeben werden.
|
||||
###############################################################################
|
||||
sub DbLog_attrLong2Array {
|
||||
my $content = shift;
|
||||
my $sptchar = shift // q{,};
|
||||
|
||||
return if(!$content);
|
||||
|
||||
my @v = map { my $p = $_; $p =~ s/\s//xg; $p; } split /$sptchar/xs, $content; ## no critic 'Map blocks'
|
||||
|
||||
return @v;
|
||||
}
|
||||
|
||||
################################################################
|
||||
# benutzte DB-Feldlängen in Helper und Internals setzen
|
||||
################################################################
|
||||
@ -8562,14 +8609,28 @@ return;
|
||||
|
||||
<b>Note:</b> <br>
|
||||
In case of fresh installed MySQL/MariaDB system don't forget deleting the anonymous "Everyone"-User with an admin-tool if
|
||||
existing !
|
||||
existing.
|
||||
<br><br>
|
||||
|
||||
Sample code and Scripts to prepare a MySQL/PostgreSQL/SQLite database you can find in
|
||||
<a href="https://svn.fhem.de/trac/browser/trunk/fhem/contrib/dblog">SVN -> contrib/dblog/db_create_<DBType>.sql</a>. <br>
|
||||
(<b>Caution:</b> The local FHEM-Installation subdirectory ./contrib/dblog doesn't contain the freshest scripts !!)
|
||||
(<b>Caution:</b> The local FHEM-Installation subdirectory ./contrib/dblog doesn't contain the freshest scripts!)
|
||||
<br><br>
|
||||
|
||||
The default installation of the MySQL/MariaDB database provides for the use of the <b>utf8_bin</b> collation.
|
||||
With this setting, characters up to 3 bytes long can be stored, which is generally sufficient.
|
||||
However, if characters with a length of 4 bytes (e.g. emojis) are to be stored in the database, the <b>utf8mb4</b>
|
||||
character set must be used. <br>
|
||||
Accordingly, in this case the MySQL/MariaDB database would be created with the following statement: <br><br>
|
||||
|
||||
<ul>
|
||||
<code> CREATE DATABASE `fhem` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; </code>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
In the configuration file (see below) utf8 support must be enabled with the key <b>utf8 => 1</b> if utf8 is to be
|
||||
used. <br><br>
|
||||
|
||||
The database contains two tables: <code>current</code> and <code>history</code>. <br>
|
||||
The latter contains all events whereas the former only contains the last event for any given reading and device.
|
||||
Please consider the <a href="#DbLog-attr-DbLogType">DbLogType</a> implicitly to determine the usage of tables
|
||||
@ -10440,14 +10501,28 @@ attr SMA_Energymeter DbLogValueFn
|
||||
|
||||
<b>Hinweis:</b> <br>
|
||||
Im Falle eines frisch installierten MySQL/MariaDB Systems bitte nicht vergessen die anonymen "Jeder"-Nutzer mit einem
|
||||
Admin-Tool (z.B. phpMyAdmin) zu löschen falls sie existieren !
|
||||
Admin-Tool (z.B. phpMyAdmin) zu löschen falls sie existieren.
|
||||
<br><br>
|
||||
|
||||
Beispielcode bzw. Scripts zum Erstellen einer MySQL/PostgreSQL/SQLite Datenbank ist im
|
||||
<a href="https://svn.fhem.de/trac/browser/trunk/fhem/contrib/dblog">SVN -> contrib/dblog/db_create_<DBType>.sql</a>
|
||||
enthalten. <br>
|
||||
(<b>Achtung:</b> Die lokale FHEM-Installation enthält im Unterverzeichnis ./contrib/dblog nicht die aktuellsten
|
||||
Scripte !!) <br><br>
|
||||
Scripte!) <br><br>
|
||||
|
||||
Die Standardinstallation der MySQL/MariaDB Datenbank sieht die Nutzung der Collation <b>utf8_bin</b> vor.
|
||||
Mit dieser Einstellung können Zeichen bis 3 Byte Länge gespeichert werden was im Allgemeinen ausreichend ist.
|
||||
Sollen jedoch Zeichen mit 4 Byte Länge (z.B. Emojis) in der Datenbank gespeichert werden, ist der Zeichensatz
|
||||
<b>utf8mb4</b> zu verwenden. <br>
|
||||
Dementsprechend wäre in diesem Fall die MySQL/MariaDB Datenbank mit folgendem Statement anzulegen: <br><br>
|
||||
|
||||
<ul>
|
||||
<code> CREATE DATABASE `fhem` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; </code>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
In der Konfigurationsdatei (siehe unten) ist die utf8-Unterstützung mit dem Schlüssel <b>utf8 => 1</b> einzuschalten
|
||||
sofern utf8 genutzt werden soll. <br><br>
|
||||
|
||||
Die Datenbank beinhaltet 2 Tabellen: <code>current</code> und <code>history</code>. <br>
|
||||
Die Tabelle <code>current</code> enthält den letzten Stand pro Device und Reading. <br>
|
||||
@ -10542,6 +10617,7 @@ attr SMA_Energymeter DbLogValueFn
|
||||
#);
|
||||
####################################################################################
|
||||
</pre>
|
||||
|
||||
Wird configDB genutzt, ist das Konfigurationsfile in die configDB hochzuladen ! <br><br>
|
||||
|
||||
<b>Hinweis zu Sonderzeichen:</b><br>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user