2
0
mirror of https://github.com/fhem/fhem-mirror.git synced 2025-03-10 03:06:37 +00:00

93_DbRep: dumpMySQL clientSide: add create database to dump file

git-svn-id: https://svn.fhem.de/fhem/trunk@28134 2b470e98-0d58-463d-a4d8-8e2adae1ed80
This commit is contained in:
nasseeder1 2023-11-07 19:35:13 +00:00
parent 830b9aeb44
commit bd309cccfe
2 changed files with 555 additions and 477 deletions

View File

@ -1,5 +1,6 @@
# Add changes at the top of the list. Keep it in ASCII, and 80-char wide.
# Do not insert empty lines here, update check depends on it.
- change: 93_DbRep: dumpMySQL clientSide: add create database to dump file
- change: 93_DbRep: dumpMySQL clientSide: change dump file to stricter rights
- bugfix: 47_OBIS: Implement "nohacks" attribute
- bugfix: 36_Shelly: undefinded values on restart

View File

@ -59,6 +59,7 @@ no if $] >= 5.017011, warnings => 'experimental::smartmatch';
# Version History intern
my %DbRep_vNotesIntern = (
"8.52.13" => "07.11.2023 dumpMySQL clientSide: add create database to dump file ",
"8.52.12" => "05.11.2023 dumpMySQL clientSide: change the dump file to stricter rights ",
"8.52.11" => "17.09.2023 improve the markout in func DbRep_checkValidTimeSequence, Forum:#134973 ",
"8.52.10" => "09.07.2023 fix wrong SQL syntax for PostgreSQL -> DbRep_createSelectSql, Forum:#134170 ",
@ -8249,10 +8250,7 @@ sub DbRep_mysql_DumpClientSide {
my $hash = $paref->{hash};
my $name = $paref->{name};
my $dbname = $hash->{DATABASE};
my $dump_path = AttrVal ($name, "dumpDirLocal", $dbrep_dump_path_def);
$dump_path = $dump_path."/" unless($dump_path =~ m/\/$/);
my $optimize_tables_beforedump = AttrVal ($name, "optimizeTablesBeforeDump", 0);
my $memory_limit = AttrVal ($name, "dumpMemlimit", 100000);
my $my_comment = AttrVal ($name, "dumpComment", "");
@ -8261,16 +8259,13 @@ sub DbRep_mysql_DumpClientSide {
my $ead = AttrVal ($name, "executeAfterProc", undef);
my $mysql_commentstring = "-- ";
my $character_set = "utf8";
my $repver = $hash->{HELPER}{VERSION};
my $sql_text = '';
my $sql_file = '';
my $dbpraefix = "";
my $dbname = $hash->{DATABASE};
$dump_path = $dump_path."/" unless($dump_path =~ m/\/$/);
my ($sth,$tablename,$sql_create,$rct,$insert,$first_insert,$backupfile,$drc,$drh,$e,
$sql_daten,$inhalt,$filesize,$totalrecords,$status_start,$status_end);
my (@ar,@tablerecords,@tablenames,@tables,@ergebnis);
my (%db_tables);
my ($sth,$tablename,$rct,$insert,$backupfile,$drc,$drh,$filesize,$totalrecords);
my (@ar,@tablenames,@tables,@ctab);
my (%db_tables, %db_tables_views);
my $bst = [gettimeofday]; # Background-Startzeit
@ -8279,7 +8274,7 @@ sub DbRep_mysql_DumpClientSide {
##################### Beginn Dump ########################
############################################################
undef(%db_tables);
undef %db_tables;
# Startzeit ermitteln
my ($Sekunden, $Minuten, $Stunden, $Monatstag, $Monat, $Jahr, $Wochentag, $Jahrestag, $Sommerzeit) = localtime(time);
@ -8305,42 +8300,46 @@ sub DbRep_mysql_DumpClientSide {
my @mysql_version = $sth->fetchrow;
my @v = split(/\./,$mysql_version[0]);
my $collation = '';
my $dbcharset = '';
if ($v[0] >= 5 || ($v[0] >= 4 && $v[1] >= 1) ) { # mysql Version >= 4.1
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, "SET NAMES '".$character_set."'"); # get standard encoding of MySQl-Server
return "$name|$err" if ($err);
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, "SHOW VARIABLES LIKE 'character_set_connection'");
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SHOW VARIABLES LIKE 'collation_database'));
return "$name|$err" if($err);
@ar = $sth->fetchrow;
$character_set = $ar[1];
if ($ar[1]) {
$collation = $ar[1];
$dbcharset = (split '_', $collation, 2)[0];
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SET NAMES "$dbcharset" COLLATE "$collation"));
return "$name|$err" if($err);
}
}
else { # mysql Version < 4.1 -> no SET NAMES available
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, "SHOW VARIABLES LIKE 'character_set'"); # get standard encoding of MySQl-Server
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, "SHOW VARIABLES LIKE 'dbcharset'"); # get standard encoding of MySQl-Server
return "$name|$err" if($err);
@ar = $sth->fetchrow;
if (defined($ar[1])) {
$character_set = $ar[1];
if ($ar[1]) {
$dbcharset = $ar[1];
}
}
Log3 ($name, 3, "DbRep $name - Characterset of collection set to $character_set. ");
Log3 ($name, 3, "DbRep $name - Characterset of collection set to $dbcharset. ");
undef(@tables);
undef(@tablerecords);
my %db_tables_views;
my $t = 0;
my $r = 0;
my $st_e = "\n";
my $value = 0;
my $engine = '';
my $dbpraefix = '';
my $query = "SHOW TABLE STATUS FROM `$dbname`"; # Eigenschaften der vorhandenen Tabellen ermitteln (SHOW TABLE STATUS -> Rows sind nicht exakt !!)
if ($dbpraefix ne "") {
$query .= " LIKE '$dbpraefix%'";
Log3 ($name, 3, "DbRep $name - Searching for tables inside database $dbname with prefix $dbpraefix....");
}
else {
@ -8351,7 +8350,7 @@ sub DbRep_mysql_DumpClientSide {
return "$name|$err" if($err);
while ( $value = $sth->fetchrow_hashref()) {
$value->{skip_data} = 0; #defaut -> backup data of table
$value->{skip_data} = 0; # default -> backup data of table
Log3 ($name, 5, "DbRep $name - ......... Table definition found: .........");
@ -8426,7 +8425,7 @@ sub DbRep_mysql_DumpClientSide {
return $err if($err);
}
$st_e .= "-- TABLE-INFO\n"; # Tabelleneigenschaften für SQL-File ermitteln
my $part = ''; # Tabelleneigenschaften für SQL-File ermitteln
for $tablename (@tablenames) {
my $dump_table = 1;
@ -8437,24 +8436,28 @@ sub DbRep_mysql_DumpClientSide {
}
}
if ($dump_table == 1) { # how many rows
$sql_create = "SELECT count(*) FROM `$tablename`";
if ($dump_table == 1) {
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, $sql_create);
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SELECT count(*) FROM `$tablename`));
return "$name|$err" if($err);
$db_tables{$tablename}{Rows} = $sth->fetchrow;
$db_tables{$tablename}{Rows} = $sth->fetchrow; # how many rows
$sth->finish;
$r += $db_tables{$tablename}{Rows};
push(@tables,$db_tables{$tablename}{Name}); # add tablename to backuped tables
push @tables, $db_tables{$tablename}{Name}; # add tablename to backuped tables
$t++;
if (!defined $db_tables{$tablename}{Update_time}) {
$db_tables{$tablename}{Update_time} = 0;
}
$st_e .= $mysql_commentstring."TABLE: $db_tables{$tablename}{Name} | Rows: $db_tables{$tablename}{Rows} | Length: ".($db_tables{$tablename}{Data_length}+$db_tables{$tablename}{Index_length})." | Engine: $db_tables{$tablename}{Engine}\n";
$part .= $mysql_commentstring;
$part .= "TABLE: $db_tables{$tablename}{Name} | ";
$part .= "Rows: $db_tables{$tablename}{Rows} | ";
$part .= "Length: ".($db_tables{$tablename}{Data_length} + $db_tables{$tablename}{Index_length})." | ";
$part .= "Engine: $db_tables{$tablename}{Engine}";
$part .= "\n";
if ($db_tables{$tablename}{Name} eq "current") {
$drc = $db_tables{$tablename}{Rows};
@ -8466,28 +8469,40 @@ sub DbRep_mysql_DumpClientSide {
}
}
$st_e .= "-- EOF TABLE-INFO";
$part .= $mysql_commentstring."EOF TABLE-INFO";
Log3 ($name, 3, "DbRep $name - Found ".(@tables)." tables with $r records.");
# AUFBAU der Statuszeile in SQL-File:
# -- Status | tabellenzahl | datensaetze | Datenbankname | Kommentar | MySQLVersion | Charset | EXTINFO
#
$status_start = $mysql_commentstring."Status | Tables: $t | Rows: $r ";
$status_end = "| DB: $dbname | Comment: $my_comment | MySQL-Version: $mysql_version[0] ";
$status_end .= "| Charset: $character_set $st_e\n".
$mysql_commentstring."Dump created on $CTIME_String by DbRep-Version $repver\n".$mysql_commentstring;
## Headerzeilen aufbauen
##########################
my $sql_text = $mysql_commentstring."DB Name: $dbname";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."DB Character set: $dbcharset";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."MySQL Version: $mysql_version[0]";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."Dump created on $CTIME_String by DbRep-Version $repver";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."Comment: $my_comment";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."TABLE-INFO";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring."TABLES: $t, Rows: $r";
$sql_text .= "\n";
$sql_text .= $part;
$sql_text .= "\n\n";
$sql_text = $status_start.$status_end;
## neues SQL Ausgabefile mit Header anlegen
#############################################
my $sql_file = '';
# neues SQL-Ausgabefile anlegen
($err, $sql_text, $first_insert, $sql_file, $backupfile) = DbRep_NewDumpFilename ( { sql_text => $sql_text,
($err, $sql_file, $backupfile) = DbRep_NewDumpFilename ( { sql_text => $sql_text,
dump_path => $dump_path,
dbname => $dbname,
time_stamp => $time_stamp,
character_set => $character_set
time_stamp => $time_stamp
}
);
if ($err) {
Log3 ($name, 2, "DbRep $name - $err");
$err = encode_base64 ($err, "");
@ -8497,56 +8512,109 @@ sub DbRep_mysql_DumpClientSide {
Log3 ($name, 5, "DbRep $name - New dump file $sql_file was created");
}
##################### jede einzelne Tabelle dumpen ########################
my $first_insert = 0;
## DB Einstellungen
#####################
$sql_text = "/*!40101 SET NAMES '".$dbcharset."' */;";
$sql_text .= "\n";
$sql_text .= "SET FOREIGN_KEY_CHECKS=0;";
$sql_text .= "\n\n";
DbRep_WriteToDumpFile ($sql_text, $sql_file);
## DB Create Statement einfügen
#################################
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SHOW CREATE DATABASE IF NOT EXISTS $dbname));
return "$name|$err" if($err);
my $db_create = $sth->fetchrow;
$sth->finish;
$sql_text = $mysql_commentstring;
$sql_text .= "\n";
$sql_text .= $mysql_commentstring;
$sql_text .= "Create database";
$sql_text .= "\n";
$sql_text .= $mysql_commentstring;
$sql_text .= "\n";
$sql_text .= $db_create.';';
$sql_text .= "\n";
$sql_text .= "USE `$dbname`;";
$sql_text .= "\n\n";
DbRep_WriteToDumpFile ($sql_text, $sql_file);
## jede einzelne Tabelle dumpen
#################################
$totalrecords = 0;
$sql_text = "";
for $tablename (@tables) { # first get CREATE TABLE Statement
if ($dbpraefix eq "" || ($dbpraefix ne "" && substr($tablename, 0, length($dbpraefix)) eq $dbpraefix)) {
Log3 ($name, 3, "DbRep $name - Dumping table $tablename (Type ".$db_tables{$tablename}{Engine}."):");
$a = "\n\n$mysql_commentstring\n$mysql_commentstring"."Table structure for table `$tablename`\n$mysql_commentstring\n";
$part = $mysql_commentstring;
$part .= "\n";
$part .= $mysql_commentstring;
$part .= "Table structure of table `$tablename`";
$part .= "\n";
$part .= $mysql_commentstring;
$part .= "\n";
if ($db_tables{$tablename}{Engine} ne 'VIEW' ) {
$a .= "DROP TABLE IF EXISTS `$tablename`;\n";
$part .= "DROP TABLE IF EXISTS `$tablename`;";
}
else {
$a .= "DROP VIEW IF EXISTS `$tablename`;\n";
$part .= "DROP VIEW IF EXISTS `$tablename`;";
}
$sql_text .= $a;
$sql_create = "SHOW CREATE TABLE `$tablename`";
$sql_text .= $part;
$sql_text .= "\n";
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, $sql_create);
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SHOW CREATE TABLE `$tablename`));
return "$name|$err" if($err);
@ergebnis = $sth->fetchrow;
@ctab = $sth->fetchrow;
$sth->finish;
$a = $ergebnis[1].";\n";
if (length($a) < 10) {
$err = "Fatal error! Couldn't read CREATE-Statement of table `$tablename`! This backup might be incomplete! Check your database for errors. MySQL-Error: ".$DBI::errstr;
$part = $ctab[1].";";
$part .= "\n";
if (length($part) < 10) {
$err = "Fatal error! Couldn't read CREATE-Statement for table `$tablename`! This backup might be incomplete! Check your database for errors. MySQL-Error: ".$DBI::errstr;
Log3 ($name, 2, "DbRep $name - $err");
return "$name|$err";
}
else {
$sql_text .= $a;
Log3 ($name, 5, "DbRep $name - Create-SQL found:\n$a");
$sql_text .= $part;
}
Log3 ($name, 5, "DbRep $name - Create-SQL found:\n$part");
if ($db_tables{$tablename}{skip_data} == 0) {
$sql_text .= "\n$mysql_commentstring\n$mysql_commentstring"."Dumping data for table `$tablename`\n$mysql_commentstring\n";
$sql_text .= "\n";
$sql_text .= "$mysql_commentstring\n";
$sql_text .= "$mysql_commentstring";
$sql_text .= "Dumping data of table `$tablename`\n";
$sql_text .= "$mysql_commentstring\n";
$sql_text .= "/*!40000 ALTER TABLE `$tablename` DISABLE KEYS */;";
DbRep_WriteToDumpFile ($sql_text, $sql_file);
$sql_text = "";
$fieldlist = "(";
$sql_create = "SHOW FIELDS FROM `$tablename`"; # build fieldlist
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, $sql_create);
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, qq(SHOW FIELDS FROM `$tablename`));
return "$name|$err" if($err);
while (@ar = $sth->fetchrow) {
$fieldlist = "(";
while (@ar = $sth->fetchrow) { # build fieldlist
$fieldlist .= "`".$ar[0]."`,";
}
@ -8564,25 +8632,25 @@ sub DbRep_mysql_DumpClientSide {
$insert = "INSERT INTO `$tablename` $fieldlist VALUES ("; # default beginning for INSERT-String
$first_insert = 0;
$sql_daten = "SELECT * FROM `$tablename` LIMIT ".$ttt.",".$dumpspeed.";"; # get rows (parts)
my $sql_daten = "SELECT * FROM `$tablename` LIMIT ".$ttt.",".$dumpspeed.";"; # get rows (parts)
($err, $sth) = DbRep_prepareExecuteQuery ($name, $dbh, $sql_daten);
return "$name|$err" if($err);
while ( @ar = $sth->fetchrow) { # Start the insert
if ($first_insert == 0) {
$a = "\n$insert";
$part = "\n$insert";
}
else {
$a = "\n(";
$part = "\n(";
}
for $inhalt(@ar) { # quote all values
$a .= $dbh->quote($inhalt).",";
for my $cont (@ar) { # quote all values
$part .= $dbh->quote($cont).",";
}
$a = substr($a,0, length($a)-1).");"; # remove trailing ',' and add end-sql
$sql_text .= $a;
$part = substr ($part, 0, length($part)-1).");"; # remove trailing ',' and add end-sql
$sql_text .= $part;
if ($memory_limit > 0 && length($sql_text) > $memory_limit) {
($err, $filesize) = DbRep_WriteToDumpFile ($sql_text, $sql_file);
@ -8590,12 +8658,15 @@ sub DbRep_mysql_DumpClientSide {
$sql_text = "";
}
}
$sth->finish;
}
$sql_text .= "\n/*!40000 ALTER TABLE `$tablename` ENABLE KEYS */;\n";
}
($err, $filesize) = DbRep_WriteToDumpFile ($sql_text, $sql_file); # write sql commands to file
$sql_text = "";
if ($db_tables{$tablename}{skip_data} == 0) {
@ -8609,16 +8680,17 @@ sub DbRep_mysql_DumpClientSide {
}
# end
DbRep_WriteToDumpFile("\nSET FOREIGN_KEY_CHECKS=1;\n", $sql_file);
($err, $filesize) = DbRep_WriteToDumpFile ($mysql_commentstring."EOB\n", $sql_file);
# Datenbankverbindung schliessen
$sth->finish();
$dbh->disconnect();
my $rt = tv_interval($st); # SQL-Laufzeit ermitteln
my $compress = AttrVal ($name, "dumpCompress", 0); # Dumpfile komprimieren wenn dumpCompress=1
if ($compress) {
($err, $backupfile, $filesize) = DbRep_dumpCompress ($hash, $backupfile);
}
@ -9214,6 +9286,7 @@ sub DbRep_mysql_RestoreClientSide {
if ($bfile =~ m/.*.gzip$/) { # Dumpfile dekomprimieren wenn gzip
($err,$bfile) = DbRep_dumpUnCompress($hash,$bfile);
if ($err) {
$err = encode_base64($err,"");
$dbh->disconnect;
@ -9242,7 +9315,7 @@ sub DbRep_mysql_RestoreClientSide {
while (<FH>) {
$tmp = $_;
chomp($tmp);
chomp $tmp;
if (!$tmp || substr($tmp, 0, 2) eq "--") {
next;
@ -9252,17 +9325,21 @@ sub DbRep_mysql_RestoreClientSide {
if (substr($line,-1) eq ";") {
if ($line !~ /^INSERT INTO.*$/) {
Log3 ($name, 4, "DbRep $name - do query: $line");
eval { $dbh->do($line);
};
if ($@) {
}
or do {
$e = $@;
$err = encode_base64($e,"");
Log3 ($name, 1, "DbRep $name - last query: $line");
Log3 ($name, 1, "DbRep $name - $e");
close(FH);
$dbh->disconnect;
Log3 ($name, 1, "DbRep $name - last query: $line");
Log3 ($name, 1, "DbRep $name - $e");
return "$name|$err";
}
};
$line = '';
next;
@ -9297,16 +9374,18 @@ sub DbRep_mysql_RestoreClientSide {
$query = $query.";";
eval { $dbh->do($query);
};
if ($@) {
}
or do {
$e = $@;
$err = encode_base64($e,"");
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
close(FH);
$dbh->disconnect;
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
return "$name|$err";
}
};
$i = 0;
$line =~ /INSERT INTO (.*) VALUES \((.*)\);/;
@ -9323,17 +9402,20 @@ sub DbRep_mysql_RestoreClientSide {
if (length($query) >= $i_max) {
$query = $query.";";
eval { $dbh->do($query);
};
if ($@) {
}
or do {
$e = $@;
$err = encode_base64($e,"");
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
close(FH);
$dbh->disconnect;
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
return "$name|$err";
}
};
$i = 0;
$query = '';
@ -9345,16 +9427,18 @@ sub DbRep_mysql_RestoreClientSide {
}
eval { $dbh->do($query) if($i);
};
if ($@) {
}
or do {
$e = $@;
$err = encode_base64($e,"");
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
close(FH);
$dbh->disconnect;
Log3 ($name, 1, "DbRep $name - last query: $query");
Log3 ($name, 1, "DbRep $name - $e");
return "$name|$err";
}
};
$dbh->disconnect;
close(FH);
@ -12691,24 +12775,17 @@ sub DbRep_NewDumpFilename {
my $dump_path = $paref->{dump_path};
my $dbname = $paref->{dbname};
my $time_stamp = $paref->{time_stamp};
my $character_set = $paref->{character_set};
my $part = "";
my $sql_file = $dump_path.$dbname."_".$time_stamp.$part.".sql";
my $backupfile = $dbname."_".$time_stamp.$part.".sql";
$sql_text .= "/*!40101 SET NAMES '".$character_set."' */;\n";
$sql_text .= "SET FOREIGN_KEY_CHECKS=0;\n";
my ($err, $filesize) = DbRep_WriteToDumpFile ($sql_text, $sql_file);
return $err if($err);
chmod (0664, $sql_file);
$sql_text = "";
my $first_insert = 0;
return ($err, $sql_text, $first_insert, $sql_file, $backupfile);
return ($err, $sql_file, $backupfile);
}
####################################################################################################