From 40c0ced777c785fd02276aa614d213ddb860e7aa Mon Sep 17 00:00:00 2001 From: jensb <> Date: Sun, 20 Dec 2020 19:28:20 +0000 Subject: [PATCH] 55_DWD_OpenData.pm: fix forecast rotation at daybreak (forum #83097 msg #1108423) git-svn-id: https://svn.fhem.de/fhem/trunk@23392 2b470e98-0d58-463d-a4d8-8e2adae1ed80 --- fhem/CHANGED | 3 +- fhem/FHEM/55_DWD_OpenData.pm | 161 +++++++++++++++++++++++++++-------- 2 files changed, 127 insertions(+), 37 deletions(-) diff --git a/fhem/CHANGED b/fhem/CHANGED index ae2fdad3d..c5bb2f854 100644 --- a/fhem/CHANGED +++ b/fhem/CHANGED @@ -1,11 +1,12 @@ # Add changes at the top of the list. Keep it in ASCII, and 80-char wide. # Do not insert empty lines here, update check depends on it. + - bugfix: 55_DWD_OpenData: forecast rotation (forum #83097 msg #1108423) - feature: 02_RSS: added readings to show filename and type of background - bugfix: 57_SSCal: accept calendar names which are contain umlauts - feature: 98_weekprofile: add support for MQTT2_DEVICE (txl Beta-User) - feature: 89_FULLY: Added support for sensor information - bugfix: 98_weekprofile: HMCCU-HM send profile without prefix (Forum #1104569) - - bugfix: 70_DENON_AVR: serial connection issues + - bugfix: 70_DENON_AVR: serial connection issues - feature: 77_SMAEM: attr serialNumber may contain multiple serial numbers - feature: 57_SSCal: handle multiline code in description field if automatic creation of at-devices is used diff --git a/fhem/FHEM/55_DWD_OpenData.pm b/fhem/FHEM/55_DWD_OpenData.pm index d67aa7a58..b25b6aceb 100644 --- a/fhem/FHEM/55_DWD_OpenData.pm +++ b/fhem/FHEM/55_DWD_OpenData.pm @@ -35,23 +35,24 @@ Julian date conversion: Copyright (C) 2012 E. G. Richards see Explanatory Supplement to the Astronomical Almanac, 3rd edition, S.E Urban and P.K. Seidelmann eds., chapter 15.11.3, Interconverting Dates and Julian Day Numbers, Algorithm 4 -This script is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -The GNU General Public License can be found at - -http://www.gnu.org/copyleft/gpl.html. - -A copy is found in the textfile GPL.txt and important notices to the license -from the author is found in LICENSE.txt distributed with these scripts. +This script is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. This script is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +You should have received a copy of the GNU General Public License +along with this script; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +A copy of the GNU General Public License, Version 2 can also be found at + +http://www.gnu.org/licenses/old-licenses/gpl-2.0. + This copyright notice MUST APPEAR in all copies of the script! =cut @@ -616,7 +617,7 @@ use constant UPDATE_COMMUNEUNIONS => -2; use constant UPDATE_ALL => -3; require Exporter; -our $VERSION = '1.014006'; +our $VERSION = '1.016001'; our @ISA = qw(Exporter); our @EXPORT = qw(GetForecast GetAlerts UpdateAlerts UPDATE_DISTRICTS UPDATE_COMMUNEUNIONS UPDATE_ALL); our @EXPORT_OK = qw(IsCommuneUnionWarncellId); @@ -850,7 +851,7 @@ sub Shutdown { delete($hash->{".alertsFile".$communeUnion}); } } - + if (defined($hash->{".forecastBlockingCall"})) { ::BlockingKill($hash->{".forecastBlockingCall"}); } @@ -1399,7 +1400,7 @@ sub RotateForecast { while (defined(::ReadingsVal($name, 'fc'.$daysAvailable.'_date', undef))) { $daysAvailable++; } - ::Log3 $name, 5, "$name: RotateForecast: $daysAvailable days exist with readings"; + ::Log3 $name, 5, "$name: RotateForecast: START $daysAvailable day(s) exist"; my $oT = ::ReadingsVal($name, 'fc0_date', undef); my $oldToday = defined($oT)? ParseDateLocal($hash, $oT) : undef; @@ -1407,6 +1408,7 @@ sub RotateForecast { my $stationChanged = ::ReadingsVal($name, 'fc_station', '') ne $station; if ($stationChanged) { # different station, delete all existing readings + ::Log3 $name, 3, "$name: RotateForecast: station has changed, deleting exisiting readings"; ::CommandDeleteReading(undef, "$name ^fc.*"); $daysAvailable = 0; } elsif (defined($oldToday)) { @@ -1418,7 +1420,7 @@ sub RotateForecast { } my $daysForward = sprintf("%0.0f", ($today - $oldToday)/86400.0); # round() [s] -> [d] - ::Log3 $name, 5, "$name: RotateForecast: shifting forward by $daysForward day(s) ($oldToday -> $today)"; + ::Log3 $name, 3, "$name: RotateForecast: shifting forward by $daysForward day(s) ($oldToday -> $today)"; if ($daysForward > 0) { # different day if ($daysForward < $daysAvailable) { @@ -1437,16 +1439,24 @@ sub RotateForecast { push(@shiftProperties, $s.'_time'); push(@shiftProperties, $s.'_wwd'); } - # shift readings forward by days + # shift readings forward by days keeping reading timestamps for (my $d=0; $d<($daysAvailable - $daysForward); $d++) { my $sourcePrefix = 'fc'.($daysForward + $d).'_'; my $destinationPrefix = 'fc'.$d.'_'; foreach my $property (@shiftProperties) { - my $value = ::ReadingsVal($name, $sourcePrefix.$property, undef); - if (defined($value)) { - ::readingsBulkUpdate($hash, $destinationPrefix.$property, $value); + my $sourceReading = $sourcePrefix.$property; + my $destinationReading = $destinationPrefix.$property; + my $sourceValue = ::ReadingsVal($name, $sourceReading, undef); + if (defined($sourceValue)) { + my $timestamp = $hash->{READINGS}{$sourceReading}{TIME}; + ::readingsBulkUpdate($hash, $destinationReading, $sourceValue); + $hash->{READINGS}{$destinationReading}{TIME} = $timestamp; } else { - ::CommandDeleteReading(undef, $destinationPrefix.$property); + my $destinationValue = ::ReadingsVal($name, $destinationReading, undef); + if (defined($destinationValue)) { + ::Log3 $name, 3, "$name: RotateForecast WARNING: deleting reading $destinationReading because the source value $sourceReading is undefined"; + ::CommandDeleteReading(undef, "$name $destinationReading"); + } } } } @@ -1457,15 +1467,78 @@ sub RotateForecast { $daysAvailable -= $daysForward; } else { # nothing remains after shifting, delete existing day readings + ::Log3 $name, 3, "$name: RotateForecast WARNING: deleting all readings because no forecast data remains for rotation"; ::CommandDeleteReading(undef, "$name ^fc\\d+.*"); $daysAvailable = 0; } } } + ::Log3 $name, 5, "$name: RotateForecast: END $daysAvailable day(s) remain"; + return $daysAvailable; } +=head2 PruneForecast($) + +find youngest reading of each day and delete all readings that are older than 1 day +excluding the readings "day", "time" and "weekday" + +=over + +=item * param hash: hash of DWD_OpenData device + +=back + +=cut + +sub PruneForecast { + my ($hash) = @_; + my $name = $hash->{NAME}; + + if (::AttrVal($name, 'forecastPruning', 0) != 1) { + return; + } + + ::Log3 $name, 5, "$name: PruneForecast: START"; + + my @readingNames = (grep {/^fc/} keys %{$hash->{READINGS}}); + + # find youngest timestamp per day + my %youngestTimestamps; + foreach my $readingName (@readingNames) { + if (!($readingName =~ m/^fc\d*_(day|time|weekday)$/)) { + my @parts = $readingName =~ /^fc(\d+)_.*/; + if (scalar(@parts) == 1) { + my $relativeDay = $parts[0]; + my $timestamp = ::time_str2num($hash->{READINGS}{TIME}); + my $youngestTimestamp = $youngestTimestamps{$relativeDay}; + if (!defined($youngestTimestamp) || $timestamp > $youngestTimestamp) { + $youngestTimestamps{$relativeDay} = $timestamp; + } + } + } + } + + # delete readings that are too old + foreach my $readingName (@readingNames) { + if (!($readingName =~ m/^fc\d*_(day|time|weekday)$/)) { + my @parts = $readingName =~ /^fc(\d+)_.*/; + if (scalar(@parts) == 1) { + my $relativeDay = $parts[0]; + my $timestamp = ::time_str2num($hash->{READINGS}{TIME}); + my $youngestTimestamp = $youngestTimestamps{$relativeDay}; + if (defined($youngestTimestamp) && $timestamp < ($youngestTimestamp - 86400)) { + ::Log3 $name, 3, "$name: PruneForecast WARNING: deleting reading $readingName because it is more than 1 day older than all other readings of the same day"; + ::CommandDeleteReading(undef, "$name $readingName"); + } + } + } + } + + ::Log3 $name, 5, "$name: PruneForecast: END"; +} + sub ProcessForecast; =head2 GetForecast($$) @@ -1781,10 +1854,10 @@ sub ProcessForecast { my @parts = split(/ at |\n/, $@); # discard anything after " at " or newline if (@parts) { $errorMessage = $parts[0]; - ::Log3 $name, 4, "$name: ProcessForecast error: $parts[0]"; + ::Log3 $name, 4, "$name: ProcessForecast ERROR: $parts[0]"; } else { $errorMessage = $@; - ::Log3 $name, 4, "$name: ProcessForecast error: $@"; + ::Log3 $name, 4, "$name: ProcessForecast ERROR: $@"; } } else { # forecast parsed successfully @@ -1802,7 +1875,7 @@ sub ProcessForecast { } } else { $errorMessage = 'result file name not defined'; - ::Log3 $name, 3, "$name: ProcessForecast error: temp file name not defined"; + ::Log3 $name, 3, "$name: ProcessForecast ERROR: temp file name not defined"; } } @@ -1843,7 +1916,7 @@ sub GetForecastFinish { # error, skip further processing } elsif (!defined($hash->{".forecastFile"})) { $errorMessage = "internal temp file name missing"; - ::Log3 $name, 3, "$name: GetForecastFinish error: $errorMessage"; + ::Log3 $name, 3, "$name: GetForecastFinish ERROR: $errorMessage"; } else { # deserialize forecast my $fh = $hash->{".forecastFileHandle"}; @@ -1883,7 +1956,7 @@ sub GetForecastFinish { ::Log3 $name, 5, "$name: GetForecastFinish END"; } else { - ::Log 3, "GetForecastFinish error: device name missing"; + ::Log 3, "GetForecastFinish ERROR: device name missing"; } } @@ -1906,7 +1979,7 @@ sub GetForecastAbort { delete $hash->{".forecastBlockingCall"}; delete $hash->{forecastUpdating}; $errorMessage = "downloading and processing weather forecast data failed ($errorMessage)"; - ::Log3 $name, 3, "$name: GetForecastAbort error: $errorMessage"; + ::Log3 $name, 3, "$name: GetForecastAbort ERROR: $errorMessage"; ::readingsBeginUpdate($hash); ::readingsBulkUpdate($hash, 'state', "forecast error: $errorMessage"); @@ -1948,7 +2021,7 @@ sub UpdateForecast { ::readingsBeginUpdate($hash); - # preprocess existing time readings + # preprocess existing readings my $time = time(); my ($tSec, $tMin, $tHour, $tMday, $tMon, $tYear, $tWday, $tYday, $tIsdst) = Localtime($hash, $time); my $today = Timelocal($hash, 0, 0, 0, $tMday, $tMon, $tYear); @@ -2045,7 +2118,7 @@ sub UpdateForecast { } } - # delete existing time readings of all days that have not been written + # delete readings of all days that have not been updated if ($relativeDay >= 0 && $daysAvailable > $relativeDay + 1) { ::Log3 $name, 5, "$name: deleting days with index " . ($relativeDay + 1) . " to " . ($daysAvailable - 1); for (my $d=($relativeDay + 1); $d<$daysAvailable; $d++) { @@ -2053,6 +2126,9 @@ sub UpdateForecast { } } + # delete readings with inconsistent timestamps + PruneForecast($hash); + ::readingsBulkUpdate($hash, 'state', 'forecast updated'); ::readingsEndUpdate($hash, 1); @@ -2321,10 +2397,10 @@ sub ProcessAlerts { my @parts = split(/ at |\n/, $@); # discard anything after " at " or newline if (@parts) { $errorMessage = $parts[0]; - ::Log3 $name, 4, "$name: ProcessAlerts error: $parts[0]"; + ::Log3 $name, 4, "$name: ProcessAlerts ERROR: $parts[0]"; } else { $errorMessage = $@; - ::Log3 $name, 4, "$name: ProcessAlerts error: $@"; + ::Log3 $name, 4, "$name: ProcessAlerts ERROR: $@"; } } else { # alerts parsed successfully @@ -2343,7 +2419,7 @@ sub ProcessAlerts { } } else { $errorMessage = 'result file name not defined'; - ::Log3 $name, 3, "$name: ProcessAlerts error: temp file name not defined"; + ::Log3 $name, 3, "$name: ProcessAlerts ERROR: temp file name not defined"; } } @@ -2388,7 +2464,7 @@ sub GetAlertsFinish { # error, skip further processing } elsif (!defined($hash->{".alertsFile".$communeUnion})) { $errorMessage = "internal temp file name missing"; - ::Log3 $name, 3, "$name: GetAlertsFinish error: $errorMessage"; + ::Log3 $name, 3, "$name: GetAlertsFinish ERROR: $errorMessage"; } else { # deserialize alerts my $fh = $hash->{".alertsFileHandle".$communeUnion}; @@ -2458,7 +2534,7 @@ sub GetAlertsFinish { ::Log3 $name, 5, "$name: GetAlertsFinish END"; } else { - ::Log 3, "GetAlertsFinish error: device name missing"; + ::Log 3, "GetAlertsFinish ERROR: device name missing"; } } @@ -2483,7 +2559,7 @@ sub GetAlertsAbort { delete $hash->{".alertsBlockingCall".$communeUnion}; $alertsUpdating[$communeUnion] = undef; $errorMessage = "downloading and processing weather alerts data failed ($errorMessage)"; - ::Log3 $name, 3, "$name: GetAlertsAbort error: $errorMessage"; + ::Log3 $name, 3, "$name: GetAlertsAbort ERROR: $errorMessage"; $alertsErrorMessage[$communeUnion] = $errorMessage; if ($warncellId >= 0) { @@ -2639,7 +2715,7 @@ sub DWD_OpenData_Initialize { $hash->{GetFn} = 'DWD_OpenData::Get'; $hash->{AttrList} = 'disable:0,1 ' - .'forecastStation forecastDays forecastProperties forecastResolution:1,3,6 forecastWW2Text:0,1 ' + .'forecastStation forecastDays forecastProperties forecastResolution:1,3,6 forecastWW2Text:0,1 forecastPruning:0,1 ' .'alertArea alertLanguage:DE,EN alertExcludeEvents ' .'timezone ' .$readingFnAttributes; @@ -2653,6 +2729,13 @@ sub DWD_OpenData_Initialize { # # CHANGES # +# 03.12.2020 (version 1.16.1) jensb +# bugfix: delete destination reading if source reading is undefined when rotationg forecast at daybreak +# feature: new attribute forecastPruning to delete forecast readings that are more than 1 day older than the other readings of the same day +# +# 22.11.2020 (version 1.15.0) jensb +# feature: keep reading timestamp when rotating forecast values at daybreak +# # 17.06.2020 (version 1.14.6) jensb # bugfix: $warncellId uninitialized when shutdown before first forecast fetch # coding: prototypes removed @@ -2871,7 +2954,7 @@ sub DWD_OpenData_Initialize { Note: When value is changed all existing forecast readings will be deleted.
  • forecastProperties [<p1>[,<p2>]...], default: Tx, Tn, Tg, TTT, DD, FX1, Neff, RR6c, RRhc, Rh00, ww
    - A list of the properties available can be found here.
    + See the DWD forecast property defintions for more details.
    Notes:
    - Not all properties are available for all stations and for all hours.
    - If you remove a property from the list then already existing readings must be deleted manually in continuous mode.
    @@ -2879,6 +2962,12 @@ sub DWD_OpenData_Initialize {
  • forecastWW2Text {0|1}, default: 0
    Create additional wwd readings containing the weather code as a descriptive text in German language.

    • +
  • forecastPruning {0|1}, default: 0
    + Search for and delete forecast readings that are more then one day older then other forecast readings of the same day. Pruning will be performed after a successful forecast update.
    + Notes:
    + - Intended to maintain data consistency e.g. when a forecast station changes the reporting hour of a forecast property.
    + - Requires noticable extra computing resources and may cause side effects if your FHEM configuration depends on a reading that is deleted.
    +

    • alert related: