############################################## # # 93_DbLog.pm # written by Dr. Boris Neubert 2007-12-30 # e-mail: omega at online dot de # # modified by Tobias Faust 2012-06-26 # e-mail: tobias dot faust at online dot de # ############################################## package main; use strict; use warnings; use DBI; use Data::Dumper; sub DbLog($$$); ################################################################ sub DbLog_Initialize($) { my ($hash) = @_; $hash->{DefFn} = "DbLog_Define"; $hash->{UndefFn} = "DbLog_Undef"; $hash->{NotifyFn} = "DbLog_Log"; $hash->{GetFn} = "DbLog_Get"; $hash->{AttrFn} = "DbLog_Attr"; $hash->{AttrList} = "disable:0,1"; } ############################################################### sub DbLog_Define($@) { my ($hash, $def) = @_; my @a = split("[ \t][ \t]*", $def); return "wrong syntax: define DbLog configuration regexp" if(int(@a) != 4); my $regexp = $a[3]; eval { "Hallo" =~ m/^$regexp$/ }; return "Bad regexp: $@" if($@); $hash->{REGEXP} = $regexp; $hash->{CONFIGURATION}= $a[2]; return "Can't connect to database." if(!DbLog_Connect($hash)); $hash->{STATE} = "active"; return undef; } ##################################### sub DbLog_Undef($$) { my ($hash, $name) = @_; my $dbh= $hash->{DBH}; $dbh->disconnect() if(defined($dbh)); return undef; } ################################################################ # # Wird bei jeder ?nderung eines Attributes dieser # DbLog-Instanz aufgerufen # ################################################################ sub DbLog_Attr(@) { my @a = @_; my $do = 0; if($a[0] eq "set" && $a[2] eq "disable") { $do = (!defined($a[3]) || $a[3]) ? 1 : 2; } $do = 2 if($a[0] eq "del" && (!$a[2] || $a[2] eq "disable")); return if(!$do); $defs{$a[1]}{STATE} = ($do == 1 ? "disabled" : "active"); return undef; } ################################################################ # # Parsefunktion, abh?ngig vom Devicetyp # ################################################################ sub DbLog_ParseEvent($$) { my ($type, $event)= @_; my @result; # split the event into reading and argument # "day-temp: 22.0 (Celsius)" -> "day-temp", "22.0 (Celsius)" my @parts= split(/: /,$event); my $reading= $parts[0]; if(!defined($reading)) { $reading= ""; } my $arg= $parts[1]; # the interpretation of the argument depends on the device type #default my $value= $arg; if(!defined($value)) { $value= ""; } my $unit= ""; # EMEM, M232Counter, M232Voltage return plain numbers if(($type eq "M232Voltage") || ($type eq "M232Counter") || ($type eq "EMEM")) { } # FS20 elsif(($type eq "FS20") || ($type eq "X10")) { @parts= split(/ /,$value); my $reading= $parts[0]; if(!defined($reading)) { $reading= ""; } if($#parts>=1) { $value= join(" ", shift @parts); if($reading =~ m(^dim*%$)) { $value= substr($reading,3,length($reading)-4); $reading= "dim"; $unit= "%"; } else { $value= ""; } } } # FHT elsif($type eq "FHT") { if($reading =~ m(-from[12]\ ) || $reading =~ m(-to[12]\ )) { @parts= split(/ /,$event); $reading= $parts[0]; $value= $parts[1]; $unit= ""; } if($reading =~ m(-temp)) { $value=~ s/ \(Celsius\)//; $unit= "?C"; } if($reading =~ m(temp-offset)) { $value=~ s/ \(Celsius\)//; $unit= "?C"; } if($reading =~ m(^actuator[0-9]*)) { if($value eq "lime-protection") { $reading= "actuator-lime-protection"; undef $value; } elsif($value =~ m(^offset:)) { $reading= "actuator-offset"; @parts= split(/: /,$value); $value= $parts[1]; if(defined $value) { $value=~ s/%//; $value= $value*1.; $unit= "%"; } } elsif($value =~ m(^unknown_)) { @parts= split(/: /,$value); $reading= "actuator-" . $parts[0]; $value= $parts[1]; if(defined $value) { $value=~ s/%//; $value= $value*1.; $unit= "%"; } } elsif($value eq "synctime") { $reading= "actuator-synctime"; undef $value; } elsif($value eq "test") { $reading= "actuator-test"; undef $value; } elsif($value eq "pair") { $reading= "actuator-pair"; undef $value; } else { $value=~ s/%//; $value= $value*1.; $unit= "%"; } } } # KS300 elsif($type eq "KS300") { if($event =~ m(T:.*)) { $reading= "data"; $value= $event; } if($event =~ m(avg_day)) { $reading= "data"; $value= $event; } if($event =~ m(avg_month)) { $reading= "data"; $value= $event; } if($reading eq "temperature") { $value=~ s/ \(Celsius\)//; $unit= "?C"; } if($reading eq "wind") { $value=~ s/ \(km\/h\)//; $unit= "km/h"; } if($reading eq "rain") { $value=~ s/ \(l\/m2\)//; $unit= "l/m2"; } if($reading eq "rain_raw") { $value=~ s/ \(counter\)//; $unit= ""; } if($reading eq "humidity") { $value=~ s/ \(\%\)//; $unit= "%"; } if($reading eq "israining") { $value=~ s/ \(yes\/no\)//; $value=~ s/no/0/; $value=~ s/yes/1/; } } # HMS elsif($type eq "HMS") { if($event =~ m(T:.*)) { $reading= "data"; $value= $event; } if($reading eq "temperature") { $value=~ s/ \(Celsius\)//; $unit= "?C"; } if($reading eq "humidity") { $value=~ s/ \(\%\)//; $unit= "%"; } if($reading eq "battery") { $value=~ s/ok/1/; $value=~ s/replaced/1/; $value=~ s/empty/0/; } } # CUL_WS elsif($type eq "CUL_WS") { if($event =~ m(T:.*)) { $reading= "data"; $value= $event; } if($reading eq "temperature") { $unit= "?C"; } if($reading eq "humidity") { $unit= "%"; } } # BS elsif($type eq "BS") { if($event =~ m(brightness:.*)) { @parts= split(/ /,$event); $reading= "lux"; $value= $parts[4]*1.; $unit= "lux"; } } # Default else { $reading= "data"; $value= $event; } @result= ($reading,$value,$unit); return @result; } ################################################################ # # Hauptroutine zum Loggen. Wird bei jedem Eventchange # aufgerufen # ################################################################ sub DbLog_Log($$) { # Log is my entry, Dev is the entry of the changed device my ($log, $dev) = @_; return undef if($log->{STATE} eq "disabled"); # name and type required for parsing my $n= $dev->{NAME}; my $t= uc($dev->{TYPE}); # timestamp in SQL format YYYY-MM-DD hh:mm:ss #my ($sec,$min,$hr,$day,$mon,$yr,$wday,$yday,$isdst)= localtime(time); #my $ts= sprintf("%04d-%02d-%02d %02d:%02d:%02d", $yr+1900,$mon+1,$day,$hr,$min,$sec); my $re = $log->{REGEXP}; my $max = int(@{$dev->{CHANGED}}); for (my $i = 0; $i < $max; $i++) { my $s = $dev->{CHANGED}[$i]; $s = "" if(!defined($s)); if($n =~ m/^$re$/ || "$n:$s" =~ m/^$re$/) { my $ts = TimeNow(); $ts = $dev->{CHANGETIME}[$i] if(defined($dev->{CHANGETIME}[$i])); # $ts is in SQL format YYYY-MM-DD hh:mm:ss my @r= DbLog_ParseEvent($t, $s); my $reading= $r[0]; my $value= $r[1]; my $unit= $r[2]; if(!defined $reading) { $reading= ""; } if(!defined $value) { $value= ""; } if(!defined $unit || $unit eq "") { $unit = AttrVal("$n", "unit", ""); } my $is= "(TIMESTAMP, DEVICE, TYPE, EVENT, READING, VALUE, UNIT) VALUES " . "('$ts', '$n', '$t', '$s', '$reading', '$value', '$unit')"; DbLog_ExecSQL($log, "INSERT INTO history" . $is); DbLog_ExecSQL($log, "DELETE FROM current WHERE (DEVICE='$n') AND (READING='$reading')"); DbLog_ExecSQL($log, "INSERT INTO current" . $is); } } return ""; } ################################################################ # # zerlegt ?bergebenes FHEM-Datum in die einzelnen Bestandteile # und f?gt noch Defaultwerte ein # ################################################################ sub DbLog_explode_datetime($%) { my ($t, %def) = @_; my %retv; my (@datetime, @date, @time); @datetime = split("_", $t); #Datum und Zeit auftrennen @date = split("-", $datetime[0]); @time = split(":", $datetime[1]) if ($datetime[1]); if ($date[0]) {$retv{year} = $date[0];} else {$retv{year} = $def{year};} if ($date[1]) {$retv{month} = $date[1];} else {$retv{month} = $def{month};} if ($date[2]) {$retv{day} = $date[2];} else {$retv{day} = $def{day};} if ($time[0]) {$retv{hour} = $time[0];} else {$retv{hour} = $def{hour};} if ($time[1]) {$retv{minute}= $time[1];} else {$retv{minute}= $def{minute};} if ($time[2]) {$retv{second}= $time[2];} else {$retv{second}= $def{second};} $retv{datetime}=DbLog_implode_datetime($retv{year}, $retv{month}, $retv{day}, $retv{hour}, $retv{minute}, $retv{second}); #Log 1, Dumper(%retv); return %retv } sub DbLog_implode_datetime($$$$$$) { my ($year, $month, $day, $hour, $minute, $second) = @_; my $retv = $year."-".$month."-".$day."_".$hour.":".$minute.":".$second; return $retv; } ################################################################ # # Verbindung zur DB aufbauen # ################################################################ sub DbLog_Connect($) { my ($hash)= @_; my $configfilename= $hash->{CONFIGURATION}; if(!open(CONFIG, $configfilename)) { Log 1, "Cannot open database configuration file $configfilename."; return 0; } my @config=; close(CONFIG); my %dbconfig; eval join("", @config); my $dbconn= $dbconfig{connection}; my $dbuser= $dbconfig{user}; my $dbpassword= $dbconfig{password}; #check the database model if($dbconn =~ m/pg:/i) { $hash->{DBMODEL}="POSTGRESQL"; } elsif ($dbconn =~ m/mysql:/i) { $hash->{DBMODEL}="MYSQL"; } elsif ($dbconn =~ m/oracle:/i) { $hash->{DBMODEL}="ORACLE"; } else { $hash->{DBMODEL}="unknown"; Log 3, "Unknown dbmodel type in configuration file $configfilename."; Log 3, "Only Mysql, Postgresql, Oracle is currently supported."; Log 3, "Otherwise please check the connectstring: $dbconn"; return 0; } Log 3, "Connecting to database $dbconn with user $dbuser"; my $dbh = DBI->connect_cached("dbi:$dbconn", $dbuser, $dbpassword); if(!$dbh) { Log 3, "Can't connect to $dbconn: $DBI::errstr"; return 0; } Log 3, "Connection to db $dbconn established"; $hash->{DBH}= $dbh; return 1; } ################################################################ # # Prozeduren zum Ausf?hren des SQL?s # ################################################################ sub DbLog_ExecSQL1($$) { my ($dbh,$sql)= @_; my $sth = $dbh->do($sql); if(!$sth) { Log 2, "DBLog error: " . $DBI::errstr; return 0; } return $sth; } sub DbLog_ExecSQL($$) { my ($hash,$sql)= @_; Log 5, "Executing $sql"; my $dbh= $hash->{DBH}; my $sth = DbLog_ExecSQL1($dbh,$sql); if(!$sth) { #retry $dbh->disconnect(); if(!DbLog_Connect($hash)) { Log 2, "DBLog reconnect failed."; return 0; } $dbh= $hash->{DBH}; $sth = DbLog_ExecSQL1($dbh,$sql); if(!$sth) { Log 2, "DBLog retry failed."; return 0; } Log 2, "DBLog retry ok."; } return $sth; } ################################################################ # # GET Funktion # wird zb. zur Generierung der Plots implizit aufgerufen # ################################################################ sub DbLog_Get($@) { my ($hash, @a) = @_; return "Usage: get $a[0] ...\n". " where column_spec is :::\n" . " see the #DbLog entries in the .gplot files\n" . " is not used, only for compatibility for FileLog, please use - \n" . " is a prefix, - means stdout\n" if(int(@a) < 5); shift @a; my $inf = shift @a; my $outf = shift @a; my $from = shift @a; my $to = shift @a; # Now @a contains the list of column_specs my ($internal, @fld); if($outf eq "INT") { $outf = "-"; $internal = 1; } my @readings = (); my (%sqlspec, %from_datetime, %to_datetime); #uebergebenen Timestamp anpassen #m?gliche Formate: YYYY | YYYY-MM | YYYY-MM-DD | YYYY-MM-DD_HH24 %from_datetime = DbLog_explode_datetime($from, DbLog_explode_datetime("2000-01-01_00:00:00", undef)); %to_datetime = DbLog_explode_datetime($to, DbLog_explode_datetime("2099-31-12_23:59:59", undef)); $from = $from_datetime{datetime}; $to = $to_datetime{datetime}; my ($retval,$sql_timestamp,$sql_dev,$sql_reading,$sql_value) = ""; my $writeout = 0; my (@min, @max, @sum, @cnt, @lastv, @lastd); my (%tstamp, %lasttstamp, $out_tstamp, $out_value, $minval, $maxval); #fuer delta-h/d Berechnung #extract the Device:Reading arguments into @readings array for(my $i = 0; $i < int(@a); $i++) { @fld = split(":", $a[$i], 4); $readings[$i][0] = $fld[0]; $readings[$i][1] = $fld[1]; $readings[$i][2] = $fld[2]; $readings[$i][3] = $fld[3]; } my $dbh= $hash->{DBH}; #vorbereiten der DB-Abfrage, DB-Modell-abh?ngig if ($hash->{DBMODEL} eq "POSTGRESQL") { $sqlspec{get_timestamp} = "TO_CHAR(TIMESTAMP, 'YYYY-MM-DD_HH24:MI:SS')"; $sqlspec{from_timestamp} = "TO_TIMESTAMP('$from', 'YYYY-MM-DD_HH24:MI:SS')"; $sqlspec{to_timestamp} = "TO_TIMESTAMP('$to', 'YYYY-MM-DD_HH24:MI:SS')"; } elsif ($hash->{DBMODEL} eq "ORACLE") { $sqlspec{get_timestamp} = "TO_CHAR(TIMESTAMP, 'YYYY-MM-DD_HH24:MI:SS')"; $sqlspec{from_timestamp} = "TO_TIMESTAMP('$from', 'YYYY-MM-DD_HH24:MI:SS')"; $sqlspec{to_timestamp} = "TO_TIMESTAMP('$to', 'YYYY-MM-DD_HH24:MI:SS')"; } elsif ($hash->{DBMODEL} eq "MYSQL") { $sqlspec{get_timestamp} = "DATE_FORMAT(TIMESTAMP, '%Y-%m-%d_%H:%i:%s')"; $sqlspec{from_timestamp} = "STR_TO_DATE('$from', '%Y-%m-%d_%H:%i:%s')"; $sqlspec{to_timestamp} = "STR_TO_DATE('$to', '%Y-%m-%d_%H:%i:%s')"; } for(my $i=0; $i[0]."|".@readings[$i]->[1]."') AND TIMESTAMP > $sqlspec{from_timestamp} AND TIMESTAMP < $sqlspec{to_timestamp} ORDER BY TIMESTAMP"; Log 4, "DbLog: Execute Statement to Database:"; Log 4, $stm; my $sth= $dbh->prepare($stm) || return "Cannot prepare statement $stm: $DBI::errstr"; my $rc= $sth->execute() || return "Cannot execute statement $stm: $DBI::errstr"; while( ($sql_timestamp,$sql_dev,$sql_reading,$sql_value)= $sth->fetchrow_array) { $writeout = 0; $out_value = ""; $out_tstamp = ""; ############ Auswerten des 4. Parameters: function ################### if(@readings[$i]->[3] eq "int") { #nur den integerwert ?bernehmen falls zb value=15?C $out_value = $1 if($sql_value =~ m/^(\d+).*/o); $out_tstamp = $sql_timestamp; $writeout=1; } elsif (@readings[$i]->[3] =~ m/^int(\d+).*/o) { #?bernehme den Dezimalwert mit den angegebenen Stellen an Nachkommastellen $out_value = $1 if($sql_value =~ m/^([-\.\d]+).*/o); $out_tstamp = $sql_timestamp; $writeout=1; } elsif (@readings[$i]->[3] eq "delta-h") { #Berechnung eines Stundenwertes %tstamp = DbLog_explode_datetime($sql_timestamp, undef); if($lastd[$i] eq "undef") { %lasttstamp = DbLog_explode_datetime($sql_timestamp, undef); } else { %lasttstamp = DbLog_explode_datetime($lastd[$i], undef); } if("$tstamp{hour}" ne "$lasttstamp{hour}") { # Aenderung der stunde, Berechne Delta $out_value = sprintf("%0.1f", $maxval - $minval); $out_tstamp = DbLog_implode_datetime($lasttstamp{year}, $lasttstamp{month}, $lasttstamp{day}, $lasttstamp{hour}, "30", "00"); $minval = 999999; $maxval = -999999; $writeout=1; } } elsif (@readings[$i]->[3] eq "delta-d") { #Berechnung eines Tageswertes %tstamp = DbLog_explode_datetime($sql_timestamp, undef); if($lastd[$i] eq "undef") { %lasttstamp = DbLog_explode_datetime($sql_timestamp, undef); } else { %lasttstamp = DbLog_explode_datetime($lastd[$i], undef); } if("$tstamp{day}" ne "$lasttstamp{day}") { # Aenderung des Tages, Berechne Delta $out_value = sprintf("%0.1f", $maxval - $minval); $out_tstamp = DbLog_implode_datetime($lasttstamp{year}, $lasttstamp{month}, $lasttstamp{day}, "00", "00", "00"); $minval = 999999; $maxval = -999999; $writeout=1; } } elsif(@readings[$i]->[3]) { #evaluate my $value = $sql_value; $out_value = eval("@readings[$i]->[3]"); if($@) {Log 3, "DbLog: Fehler in der ?bergebenen Funktion: <".@readings[$i]->[3].">, Fehler: $@";} $out_tstamp = $sql_timestamp; $writeout=1; } else { $out_value = $sql_value; $out_tstamp = $sql_timestamp; $writeout=1; } ###################### Ausgabe ########################### $retval .= "$out_tstamp $out_value\n" if($writeout); if(defined($sql_value) || $sql_value =~ m/^[-\.\d]+$/o){ #nur setzen wenn nummerisch $min[$i] = $sql_value if($sql_value < $min[$i]); $max[$i] = $sql_value if($sql_value > $max[$i]);; $sum[$i] += $sql_value; $minval = $sql_value if($sql_value < $minval); $maxval = $sql_value if($sql_value > $maxval); } else { $min[$i] = 0; $max[$i] = 0; $sum[$i] = 0; $minval = 0; $maxval = 0; } $cnt[$i]++; $lastv[$i] = $sql_value; $lastd[$i] = $sql_timestamp; } #while fetchrow ######## den letzten Abschlusssatz rausschreiben ########## if(@readings[$i]->[3] eq "delta-h" || @readings[$i]->[3] eq "delta-d") { $out_value = sprintf("%0.1f", $maxval - $minval); $out_tstamp = DbLog_implode_datetime($lasttstamp{year}, $lasttstamp{month}, $lasttstamp{day}, $lasttstamp{hour}, "30", "00") if(@readings[$i]->[3] eq "delta-h"); $out_tstamp = DbLog_implode_datetime($lasttstamp{year}, $lasttstamp{month}, $lasttstamp{day}, "00", "00", "00") if(@readings[$i]->[3] eq "delta-d"); $retval .= "$out_tstamp $out_value\n"; } # DatenTrenner setzen $retval .= "#@readings[$i]->[0]:@readings[$i]->[1]:@readings[$i]->[2]:@readings[$i]->[3]\n"; } #for @readings #Ueberfuehren der gesammelten Werte in die globale Variable %data for(my $j=0; $j

DbLog


    Define
      define <name> DbLog <configfilename> <regexp>

      Log events to a database. The database connection is defined in <configfilename> (see sample configuration file db.conf). The configuration is stored in a separate file to avoid storing the password in the main configuration file and to have it visible in the output of the list command.

      You must have 93_DbLog.pm in the FHEM subdirectory to make this work. Additionally, the modules DBI and DBD::<dbtype> need to be installed (use cpan -i <module> if your distribution does not have it).

      <regexp> is the same as in FileLog.

      Sample code to create a MySQL database is in fhemdb_create.sql. The database contains two tables: current and history. The latter contains all events whereas the former only contains the last event for any given reading and device. The columns have the following meaning:
      1. TIMESTAMP: timestamp of event, e.g. 2007-12-30 21:45:22
      2. DEVICE: device name, e.g. Wetterstation
      3. TYPE: device type, e.g. KS300
      4. EVENT: event specification as full string, e.g. humidity: 71 (%)
      5. READING: name of reading extracted from event, e.g. humidity
      6. VALUE: actual reading extracted from event, e.g. 71
      7. UNIT: unit extracted from event, e.g. %
      The content of VALUE is optimized for automated post-processing, e.g. yes is translated to 1

      The current values can be retrieved by the following code like FileLog:
        get myDbLog - - 2012-11-10 2012-11-10 KS300:temperature::


      Examples:
        # log everything to database
        define myDbLog DbLog /etc/fhem/db.conf .*:.*
    Set
      N/A

    Get
      get <name> <infile> <outfile> <from> <to> <column_spec>

      Read data from the Database, used by frontends to plot data without direct access to the Database.
      • <infile>
        A dummy parameter for FileLog compatibility. Always set to -
      • <outfile>
        A dummy parameter for FileLog compatibility. Always set to -
      • <from> / <to>
        Used to select the data. Please use the following timeformat or an initial substring of it:
          YYYY-MM-DD_HH24:MI:SS
      • <column_spec>
        For each column_spec return a set of data separated by a comment line on the current connection.
        Syntax: <device>:<reading>:<default>:<fn>
        • <device>
          The name of the device. Case sensitive
        • <reading>
          The reading of the given device to select. Case sensitive.
        • <default>
          no implemented yet
        • <fn> One of the following:
          • int
            Extract the integer at the beginning of the string. Used e.g. for constructs like 10%
          • int<digit>
            Extract the decimal digits including negative character and decimal point at the beginning og the string. Used e.g. for constructs like 15.7°C
          • delta-h or delta-d
            Return the delta of the values for a given hour or a given day. Used if the column contains a counter, as is the case for the KS300 rain column.
          • everything else
            The string is evaluated as a perl expression. $value is the current value returned from the Database. Note: The string/perl expression cannot contain spaces, as the part after the space will be considered as the next column_spec.


      Example:
      • get myDbLog - - 2012-11-10 2012-11-20 KS300:temperature::
      • get myDbLog - - 2012-11-10_10 2012-11-10_20 KS300:temperature::int1
        like from 10am until 20pm at 10.11.2012
      • get myDbLog - - 2012-11-10 2012-11-20 KS300:temperature::
      • get myDbLog - - 2012-11-10 2012-11-20 KS300:temperature:: KS300:rain::delta-h KS300:rain::delta-d
      • get myDbLog - - 2012-11-10 2012-11-20 MyDummy:data::$value=~"on"?1:0


    Attributes
      N/A

=end html =cut