#!/usr/bin/perl -w
################################################################
# $Id$
# vim: ts=2:et
#
#  (c) 2012 Copyright: Martin Fischer (m_fischer at gmx dot de)
#  All rights reserved
#
#  This script free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  any later version.
#
#  The GNU General Public License can be found at
#  http://www.gnu.org/copyleft/gpl.html.
#  A copy is found in the textfile GPL.txt and important notices to the license
#  from the author is found in LICENSE.txt distributed with these scripts.
#
#  This script is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
################################################################
use CGI qw(:standard Vars);
use CGI::Carp qw(warningsToBrowser fatalsToBrowser);
use DBI; #requires libdbd-sqlite3-perl
use lib "./lib";
use Geo::IP;
use strict;
use warnings;
no warnings 'uninitialized';

sub createDB();
sub insertDB();
sub checkColumn($$);
sub getLocation($);
sub googleVisualizationLib($);
sub drawMarkersMap(@);
sub drawPieChart(@);
sub drawRegionsMap(@);
sub drawTable(@);
sub drawColumnChartTop10Modules(@);
sub drawColumnChartTop10ModDef(@);
sub drawBarChartModules(@);
sub viewStatistics();

# cascading style sheet
my $css = "http://fhem.de/../css/style.css";

# exclude modules from top 10 graph
my $excludeFromTop10modules = "at autocreate notify telnet weblink FileLog FHEMWEB Global SUNRISE_EL";
my $excludeFromTop10definitions = "at autocreate notify telnet weblink FileLog FHEMWEB Global SUNRISE_EL";

# directory cointains databases
my $datadir = "./data";

# geo ip database file from http://www.maxmind.com/download/geoip/database/
# should be updated once per month
my $geoIPDat = "$datadir/GeoLiteCity.dat";

# database
my $dbf = "$datadir/fhem_statistics_db.sqlite";
my $dsn = "dbi:SQLite:dbname=$dbf";
my $sth;

# fhem node
my $ua = $ENV{HTTP_USER_AGENT};
my $ip = $ENV{REMOTE_ADDR};
my %data = Vars();

# create database if not exists
createDB() if (! -e $dbf);

my $dbh = DBI->connect($dsn,"","", { RaiseError => 1, ShowErrorStatement => 1 }) ||
  die "Cannot connect: $DBI::errstr";

if(index($ua,"Fhem") > -1) {
  print header("application/x-www-form-urlencoded");
  insertDB();
  print "==> ok";
} else {
  viewStatistics();
}

sub viewStatistics() {
  my $visLib        = googleVisualizationLib("'corechart','geochart','table'");
  my $cOS           = drawPieChart("nodes","os","Operating System",390,300,"chart_os");
  my $cArch         = drawPieChart("nodes","arch","Architecture",390,300,"chart_arch");
  my $cRelease      = drawPieChart("nodes","release","FHEM Release",390,300,"chart_release");
  my $cPerl         = drawPieChart("nodes","perl","Perl Version",390,300,"chart_perl");
  my $cModulesTop10 = drawColumnChartTop10Modules("modules","modulestop10",,"Used",800,300,"chart_modulestop10");
  my $cModDefTop10  = drawColumnChartTop10ModDef("modules","definitions","Definitions",800,300,"chart_moddeftop10");
  #my $cModules      = drawBarChartModules("modules","modules","Used","Definitions",800,600,"chart_modules");
  my $mWorld        = drawRegionsMap("locations","countryname","world","map_world");
  my $mEU           = drawRegionsMap("locations","countryname","150","map_europe");
  my $mWesternEU    = drawMarkersMap("locations","city","155","map_germany");
  my $tModules      = drawTable3cols("modules","total_modules","string","Module","number","Used","number","Definitions","table_modules");
  #my $tModDef       = drawTable("modules","total_moddef","string","Module","number","Definitions","table_moddef");
  my $tModels       = drawTable2cols("models","total_models","string","Model","boolean","defined","table_models");
  my @res = $dbh->selectrow_array("SELECT created FROM db");
  my $since = "@res";

  print header;
  print start_html(
        -title  => 'fhem.de - Statistics',
        -author => 'm_fischer@gmx.de',
        -base   => 'true',
        -style  => {-src => $css},
        -meta   => {'keywords' => 'fhem houseautomation statistics'},
        -script => [
                      { -type => 'text/javascript',
                        -src  => 'https://www.google.com/jsapi',
                      },
                      $visLib,
                      $cOS, $cArch, $cRelease, $cPerl,
                      $cModulesTop10, $cModDefTop10,
                      #$cModules,
                      $mWorld, $mEU, $mWesternEU,
                      $tModules, $tModels,
                   ],
  );

  my ($nodes) = $dbh->selectrow_array("SELECT COUNT(uniqueID) FROM nodes");

  print <<END;
  <div id="logo"></div>
  <div id="menu">
    <table><tbody><tr><td>
    <table id="room">
      <tr><td></td></tr>
      <tr><td><b>back to</b></td></tr>
      <tr><td></td></tr> 
      <tr><td><a href="http://fhem.de">Homepage</a></td></tr>
      <tr><td></td></tr>
    </tbody></table>
    </td></tr></tbody></table>
  </div>

  <div id="right">
    <noscript>
      <div style="text-align:center; border: 2px solid red; background: #D7FFFF;">
        <div style="text-align:center; background: #D7FFFF; color: red;">
          <h4>Please enable Javascript on your Browser!</h4>
        </div>
      </div>
    </noscript>

    <h3>Fhem Statistics ($nodes submissions since $since)</h3>
    <h4>Installed on...</h4>
    <div id="chart_os" style="float:left; border: 1px solid black; margin-right:18px;"></div>
    <div id="chart_arch" style="float:left; border: 1px solid black;"></div>
    <div style="clear:both;"></div>

    <h4>Versions...</h4>
    <div id="chart_release" style="float:left; border: 1px solid black; margin-right:18px;"></div>
    <div id="chart_perl" style="float:left; border: 1px solid black;"></div>
    <div style="clear:both;"></div>

    <h4>Top 10 of most commonly used modules<small><sup>1</sup></small>...</h4>
    <div id="chart_modulestop10" style="width: 800px; height: 300px; border: 1px solid black;"></div>
    <small><sup>1</sup> excluded from graph: $excludeFromTop10modules</small>

    <h4>Top 10 of total definitions by module<small><sup>1</sup></small>...</h4>
    <div id="chart_moddeftop10" style="width: 800px; height: 300px; border: 1px solid black;"></div>
    <small><sup>1</sup> excluded from graph: $excludeFromTop10definitions</small>

<!--
//    <h4>Top 20 of most commonly used modules (with total definitions by module)...</h4>
//    <div id="chart_modules" style="width: 825px; height: 600px; border: 1px solid black;"></div>
//-->

    <h4>Locations worldwide...</h4>
    <div id="map_world" style="width: 800px; height: 500px; border: 1px solid black;"></div>

    <h4>Locations in Europe...</h4>
    <div id="map_europe" style="width: 800px; height: 500px; border: 1px solid black;"></div>

    <h4>Locations in Western Europe...</h4>
    <div id="map_germany" style="width: 800px; height: 500px; border: 1px solid black;"></div>

    <div style="float:left; width: 390px; margin-right:20px;">
      <h4>List of total used modules (with definitions)...</h4>
      <div id="table_modules" style="width: 390px; border: 1px solid black;"></div>
      <small><strong>Note:</strong> Click on a column header for sorting</small>
    </div>

    <div style="float:left; width: 390px;">
      <h4>List of defined models...</h4>
      <div id="table_models" style="width: 390px; border: 1px solid black;"></div>
      <small><strong>Note:</strong> Click on a column header for sorting</small>
    </div>
    <div style="clear:both;"></div>

    <div id="footer" style="position:relative; left: -100px; text-align: center;">
      <p><small>Layout by M. Fischer - Click <a href="http://www.fischer-net.de/kontakt.html">here</a> to leave your comments.</small></p>
    </div>
  </div>
END
  print end_html;
}

sub googleVisualizationLib($) {
  my $packages = shift;
  my $code =<<END;
// Load the Visualization API library
google.load('visualization', '1.0', {'packages':[$packages]});
END
  return $code;
}

sub drawPieChart(@) {
  my ($table,$column,$title,$width,$height,$divID) = @_;
  my $res = $dbh->selectall_arrayref("SELECT DISTINCT $column FROM $table ORDER BY $column ASC");

  my %hash = ();
  foreach my $row (@$res) {
    my ($value) = @$row;
    my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $table WHERE $column = '$value'");
    $hash{$value} = $count;
  }

  my $addRows;
  foreach my $value (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    $addRows .= "\t['$value',$hash{$value}],\n";
  }
  chop($addRows);

  my $code =<<END;
  google.setOnLoadCallback(drawChart_$column);

  function drawChart_$column() {
    var data = new google.visualization.DataTable();
    data.addColumn('string', 'Topping');
    data.addColumn('number', 'Slices');
    data.addRows([
    $addRows
    ]);

    var options = {
      title     : '$title',
      width     : $width,
      height    : $height,
      is3D      : true,
      tooltip   : { showColorCode: true, },
      chartArea : { height:'80%',width:'95%', },
    };

    var chart = new google.visualization.PieChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END

  return $code;
}

sub drawColumnChartTop10Modules(@) {
  my ($table,$postfix,$rowtitle,$width,$height,$divID) = @_;
  $sth = $dbh->prepare("SELECT * FROM $table where 1=0");
  $sth->execute();
  my $res = $sth->{NAME};
  $sth->finish;

  my %hash = ();
  foreach my $column (@$res) {
    #my ($sum) = $dbh->selectrow_array("SELECT sum($column) FROM $table");
    my ($sum) = $dbh->selectrow_array("SELECT count($column) FROM $table WHERE $column != 0");
    $hash{$column} = $sum;
  }

  my $data;
  my $i=0;
  foreach my $column (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    next if($column eq "uniqueID");
    next if($excludeFromTop10modules =~ /$column/);
    $data .= "\t['$column',$hash{$column}],\n";
    $i++;
    last if($i == 10);
  }
  chop($data);

  my $code =<<END;
  google.setOnLoadCallback(drawChart_$postfix);
  function drawChart_$postfix() {
    var data = google.visualization.arrayToDataTable([
      ['Module','$rowtitle'],
    $data
    ]);

    var options = {
      // title  : 'title',
      legend    : { position:'none' },
      chartArea : { width:"90%" },
      fontSize  : 12,
      vAxis     : { minValue:0, },
    };

    var chart = new google.visualization.ColumnChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END
  return $code;
}

sub drawColumnChartTop10ModDef(@) {
  my ($table,$postfix,$rowtitle,$width,$height,$divID) = @_;
  $sth = $dbh->prepare("SELECT * FROM $table where 1=0");
  $sth->execute();
  my $res = $sth->{NAME};
  $sth->finish;

  my %hash = ();
  foreach my $column (@$res) {
    my ($sum) = $dbh->selectrow_array("SELECT sum($column) FROM $table");
    $hash{$column} = $sum;
  }

  my $data;
  my $i=0;
  foreach my $column (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    next if($column eq "uniqueID");
    next if($excludeFromTop10definitions =~ /$column/);
    $data .= "\t['$column',$hash{$column}],\n";
    $i++;
    last if($i == 10);
  }
  chop($data);

  my $code =<<END;
  google.setOnLoadCallback(drawChart_$postfix);
  function drawChart_$postfix() {
    var data = google.visualization.arrayToDataTable([
      ['Module','$rowtitle'],
    $data
    ]);

    var options = {
      // title  : 'title',
      legend    : { position:'none' },
      chartArea : { width:"90%" },
      fontSize  : 12,
      vAxis     : { minValue:0, },
    };

    var chart = new google.visualization.ColumnChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END
  return $code;
}

sub drawBarChartModules(@) {
  my ($table,$postfix,$row1title,$row2title,$width,$height,$divID) = @_;
  $sth = $dbh->prepare("SELECT * FROM $table where 1=0");
  $sth->execute();
  my $res = $sth->{NAME};
  $sth->finish;

  my %hash = ();
  foreach my $column (@$res) {
    next if($column eq "uniqueID");
    my ($count) = $dbh->selectrow_array("SELECT count($column) FROM $table WHERE $column != 0");
    my ($sum)   = $dbh->selectrow_array("SELECT sum($column) FROM $table");
    $hash{$column}{count} = $count;
    $hash{$column}{sum}   = $sum;
  }

  my $data;
  my $i=0;
  foreach my $column (sort {$hash{$b}{count} <=> $hash{$a}{count}} keys %hash) {
    $data .= "\t['$column',$hash{$column}{count},$hash{$column}{sum}],\n";
    $i++;
    last if($i == 20);
  }
  chop($data);

  my $code =<<END;
  google.setOnLoadCallback(drawChart_$postfix);
  function drawChart_$postfix() {
    var data = google.visualization.arrayToDataTable([
      ['Module','$row1title','$row2title'],
    $data
    ]);

    var options = {
      height    : $height,
      width     : $width,
      chartArea : {left:150,top:20,width:"65%",height:"90%"},
      // title  : 'title',
    };

    var chart = new google.visualization.BarChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END
  return $code;
}

sub drawMarkersMap(@) {
  my ($table,$column,$region,$divID) = @_;
  my $res = $dbh->selectall_arrayref("SELECT DISTINCT $column FROM $table ORDER BY $column ASC");

  my %hash = ();
  foreach my $row (@$res) {
    my ($value) = @$row;
    my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $table WHERE $column = '$value'");
    #$value = "Germany" if($value eq "");
    next if($value eq "");
    $hash{$value} = $count;
  }

  my $addRows;
  foreach my $value (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    $addRows .= "\t['$value',$hash{$value}],\n";
  }
  chop($addRows);

  my $code=<<END;
  google.setOnLoadCallback(drawMarkersMap_$region);

  function drawMarkersMap_$region() {
    var data = google.visualization.arrayToDataTable([
      ['City','Installations'],
      $addRows
    ]);

    var options = {
      region: '$region',
      displayMode: 'markers',
      colorAxis: {colors: ['gold', 'darkgreen']},
      backgroundColor : 'lightblue',
    };

    var chart = new google.visualization.GeoChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END

  return $code;
}

sub drawRegionsMap(@) {
  my ($table,$column,$region,$divID) = @_;
  my $res = $dbh->selectall_arrayref("SELECT DISTINCT $column FROM $table ORDER BY $column ASC");

  my %hash = ();
  foreach my $row (@$res) {
    my ($value) = @$row;
    my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM $table WHERE $column = '$value'");
    $hash{$value} = $count;
  }

  my $addRows;
  foreach my $value (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    $addRows .= "\t['$value',$hash{$value}],\n";
  }
  chop($addRows);

  my $code=<<END;
  google.setOnLoadCallback(drawRegionsMap_$region);

  function drawRegionsMap_$region() {
    var data = google.visualization.arrayToDataTable([
      ['Country','Installations'],
      $addRows
    ]);

    var options = {
      region: '$region',
      // colorAxis: {colors: ['#FFFF80', 'darkgreen']},
      backgroundColor : 'lightblue',
    };

    var chart = new google.visualization.GeoChart(document.getElementById('$divID'));
    chart.draw(data, options);
  };
END
  return $code;
}

sub drawTable2cols(@) {
  my ($table,$postfix,$type1,$title1,$type2,$title2,$divID) = @_;
  $sth = $dbh->prepare("SELECT * FROM $table where 1=0");
  $sth->execute();
  my $res = $sth->{NAME};
  $sth->finish;

  my %hash = ();
  foreach my $column (@$res) {
    my ($sum) = $dbh->selectrow_array("SELECT sum(\"$column\") FROM $table");
    $hash{$column} = $sum;
  }

  my $data;
  if($type2 eq "boolean") {
    foreach my $column (sort keys %hash) {
      next if($column eq "uniqueID");
      $data .= "\t['$column',true],\n";
    }
  } else {
    foreach my $column (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
      next if($column eq "uniqueID");
      $data .= "\t['$column',$hash{$column}],\n";
    }
  }
  chop($data);

  my $code=<<END;
  google.setOnLoadCallback(drawTable_$postfix);
  function drawTable_$postfix() {
    var data = new google.visualization.DataTable();
    data.addColumn('$type1', '$title1');
    data.addColumn('$type2', '$title2');
    data.addRows([
    $data
    ]);

    var options = {
      showRowNumber : false,
      sortAscending : true,
      sortColumn    : 0,
      height        : 400,
    };

    var table = new google.visualization.Table(document.getElementById('$divID'));
    table.draw(data,options);
  };
END
  return $code;
}

sub drawTable3cols(@) {
  my ($table,$postfix,$type1,$title1,$type2,$title2,$type3,$title3,$divID) = @_;
  $sth = $dbh->prepare("SELECT * FROM $table where 1=0");
  $sth->execute();
  my $res = $sth->{NAME};
  $sth->finish;

  my %hash = ();
  foreach my $column (@$res) {
    my ($count) = $dbh->selectrow_array("SELECT count(\"$column\") FROM $table WHERE \"$column\" != 0");
    my ($sum)   = $dbh->selectrow_array("SELECT sum(\"$column\") FROM $table");
    $hash{$column}{count} = $count;
    $hash{$column}{sum}   = $sum;
  }

  my $data;
  foreach my $column (sort {$hash{$b} <=> $hash{$a}} keys %hash) {
    next if($column eq "uniqueID");
    $data .= "\t['$column',$hash{$column}{count},$hash{$column}{sum}],\n";
  }
  chop($data);

  my $code=<<END;
  google.setOnLoadCallback(drawTable_$postfix);
  function drawTable_$postfix() {
    var data = new google.visualization.DataTable();
    data.addColumn('$type1', '$title1');
    data.addColumn('$type2', '$title2');
    data.addColumn('$type3', '$title3');
    data.addRows([
    $data
    ]);

    var options = {
      showRowNumber : false,
      sortAscending : false,
      sortColumn    : 1,
      height        : 400,
    };

    var table = new google.visualization.Table(document.getElementById('$divID'));
    table.draw(data,options);
  };
END
  return $code;
}

sub createDB() {
  my $dbh = DBI->connect($dsn,"","", { RaiseError => 1, ShowErrorStatement => 1 }) ||
    die "Cannot connect: $DBI::errstr";
  $dbh->do("CREATE TABLE db (created TIMESTAMP DEFAULT CURRENT_TIMESTAMP)");
  $dbh->do("CREATE TABLE nodes (uniqueID VARCHAR(32) PRIMARY KEY UNIQUE, release VARCHAR(16), branch VARCHAR(32), os VARCHAR(32), arch VARCHAR(64), perl VARCHAR(16), lastSeen TIMESTAMP DEFAULT CURRENT_TIMESTAMP)");
  $dbh->do("CREATE TABLE locations (uniqueID VARCHAR(32) PRIMARY KEY UNIQUE, countrycode VARCHAR(2), countrycode3 VARCHAR(3), countryname VARCHAR(64), region CHAR(2) ,regionname VARCHAR(64), city VARCHAR(255), latitude FLOAT(8,6), longitude FLOAT(8,6), timezone VARCHAR(64), continentcode CHAR(2))");
  $dbh->do("CREATE TABLE modules (uniqueID VARCHAR(32) PRIMARY KEY UNIQUE)");
  $dbh->do("CREATE TABLE models (uniqueID VARCHAR(32) PRIMARY KEY UNIQUE)");
  $dbh->do("INSERT INTO db (created) VALUES (CURRENT_TIMESTAMP)");
  $dbh->disconnect();
  return;
}

sub insertDB() {
  my $uniqueID = $data{uniqueID};
  my $modules = $data{modules};
  my $models = $data{models};
  my $sth;

  # insert or update fhem node
  my ($release,$branch,$os,$arch,$perl);
  foreach (split(/\|/,$data{system})) {
    my ($k,$v) = split /:/;
    $release = $v if($k eq "Release");
    $branch  = $v if($k eq "Branch");
    $os      = $v if($k eq "OS");
    $arch    = $v if($k eq "Arch");
    $perl    = $v if($k eq "Perl");
  }
  $sth = $dbh->prepare(q{REPLACE INTO nodes (uniqueID,release,branch,os,arch,perl,lastSeen) VALUES(?,?,?,?,?,?,CURRENT_TIMESTAMP)});
  $sth->execute($uniqueID,$release,$branch,$os,$arch,$perl);

  # insert or update geo location of fhem node
#### TODO: sprachcode 84.191.75.195
  my @geo = getLocation($ip);

  if(@geo) {
    $sth = $dbh->prepare(q{REPLACE INTO locations (uniqueID,countrycode,countrycode3,countryname,region,regionname,city,latitude,longitude,timezone,continentcode) VALUES(?,?,?,?,?,?,?,?,?,?,?)});
    $sth->execute($uniqueID,$geo[0],$geo[1],$geo[2],$geo[3],$geo[4],$geo[5],$geo[6],$geo[7],$geo[8],$geo[9]);
  }

  # delete old modules of fhem node
  $sth = $dbh->prepare(q{DELETE FROM modules WHERE uniqueID=?});
  $sth->execute($uniqueID);

  # insert new modules of fhem node
  $sth = $dbh->prepare("INSERT INTO modules (uniqueID) VALUES (?)");
  $sth->execute($uniqueID);

  foreach (split(/\|/,$data{modules})) {
    my ($k,$v) = split /:/;
    checkColumn("modules",$k);

    $sth = $dbh->prepare("UPDATE modules SET '$k'='$v' WHERE uniqueID='$uniqueID'");
    $sth->execute();
  }

  if($data{models}) {
    # delete old models of fhem node
    $sth = $dbh->prepare(q{DELETE FROM models WHERE uniqueID=?});
    $sth->execute($uniqueID);

    # insert new modules of fhem node
    $sth = $dbh->prepare("INSERT INTO models (uniqueID) VALUES (?)");
    $sth->execute($uniqueID);

    foreach (split(/\|/,$data{models})) {
      my @models = split /,/;
      foreach my $m (@models) {
        checkColumn("models",$m);
        $sth = $dbh->prepare("UPDATE models SET '$m'='1' WHERE uniqueID='$uniqueID'");
        $sth->execute();
      }
    }
  }

  $sth->finish();
  $dbh->disconnect();
  
  return;
}

sub checkColumn($$) {
  my ($t,$k) = @_;

  # get table info
  my %column = %{ $dbh->column_info(undef, undef,$t, undef)->fetchall_hashref('COLUMN_NAME') };

  # check if column exists
  if(!exists $column{$k}) {
    $sth = $dbh->prepare("ALTER TABLE $t ADD COLUMN '$k' INTEGER DEFAULT 0");
    $sth->execute();
    $sth->finish;
  }

  return;
}

sub getLocation($) {
  my ($ip) = shift;
  my $gi = Geo::IP->open($geoIPDat, GEOIP_STANDARD);
  my $rec = $gi->record_by_addr($ip);

  if(!$rec) {
    return;
  } else {
    return (
      $rec->country_code,$rec->country_code3,$rec->country_name,$rec->region,$rec->region_name,$rec->city,
      $rec->latitude,$rec->longitude,$rec->time_zone,$rec->continent_code
    );
  }
}

1;