#!/bin/sh
PGNAME="nspollrawsdata"
PGDESC="NovaStar station polling for CDEC/NOAA/MESOWEST RAWS data"
PGVERS=$(nsversion)
PSQL="psql novastar novastar"
OFILE="/tmp/$PGNAME.$$.1.tmp"
TMPFILE="/tmp/$PGNAME.$$.2.tmp"
LOGFILE="/usr/ns/log/$PGNAME.log"
LOCKFILE="/usr/ns/locks/$PGNAME"
DEF_POLLTYPE=3
DEF_TIMEOUT="5"
DEF_TIMESPAN="2hours"
DEF_TIMESPANSECS="7200"
# Display program usage
usage () {
echo "$PGNAME [$PGVERS]: $PGDESC"
echo "Usage: $PGNAME [options...] -s#[,#,...]"
echo "where:"
echo " -s #,... Station IDs to poll."
echo "and [options...] are:"
echo " -d # Debug display level."
echo " -f Do NOT file interrogation results in database."
echo " -force Force polling to override time limits."
echo " -help|--help Display this help."
echo " -q|-quiet|--quiet Do not display activity messages."
echo " -t # Station polling type:"
echo " 0=current reading "
echo " 1=last sampled"
echo " 2=last logged"
echo " 3=logged data since last poll"
echo " 4=logged data between start and stop times"
echo " -ts MM/DD/YY-HH:MM:SS Request data starting at this time."
echo " -te MM/DD/YY-HH:MM:SS Request data ending at this time."
echo " -version|--version Display program version."
}
# Display program usage
version () {
echo "$PGVERS"
}
# Check for program use query
if [ "$1" = "-help" -o "$1" = "--help" ]; then
usage
exit 0
fi
# Check for program version query
if [ "$1" = "-version" -o "$1" = "--version" ]; then
version
exit 0
fi
# Get command arguments.
COMMANDLINE="$0 $*"
debug="0"
polltype=$DEF_POLLTYPE
nofile="false"
force="false";
quiet="false"
MISSING="--"
STAIDS="0"
TIMEEND=""
TIMESTART=""
TIMESPAN=""
while test $# -gt 0; do
arg="$1"
if [ "$arg" = "-s" ]; then shift; STAIDS="$1";
elif [ "${arg#-s}" != "$arg" ]; then STAIDS="${arg#-s}";
elif [ "$arg" = "-f" ]; then nofile="true";
elif [ "$arg" = "-force" -o "$arg" = "--force" ]; then force="true";
elif [ "$arg" = "-q" -o "$arg" = "-quiet" -o "$arg" = "--quiet" ]; then quiet="true";
elif [ "$arg" = "-te" ]; then shift; TIMEEND="$1"
elif [ "${arg#-te}" != "$arg" ]; then TIMEEND="${arg#-te}";
elif [ "$arg" = "-ts" ]; then shift; TIMESTART="$1"
elif [ "${arg#-ts}" != "$arg" ]; then TIMESTART="${arg#-ts}";
elif [ "$arg" = "-d" ]; then shift; debug=$1
elif [ "${arg#-d}" != "$arg" ]; then debug="${arg#-d}";
elif [ "$arg" = "-t" ]; then shift; polltype=$1
elif [ "${arg#-t}" != "$arg" ]; then polltype="${arg#-t}";
else echo "Unknown argument: $arg"; fi
shift
done
# MUST DEAL WITH TIME ZONES!!!
# Recode end time
if [ -z "$TIMEEND" ]; then TIMEEND=$(dispdate "+%m-%d-%Y+%H:%M:%S"); fi
TIMEEND=$(dispdate "$TIMEEND" "+%m-%d-%Y+%H:%M:%S")
tend=$(dispdate "$TIMEEND" -t)
timestamp=$(dispdate "$TIMEEND" "+%Y-%m-%d %H:%M:%S")
# Limit time span for current or last sample poll type
if [ $polltype -le 2 ]; then
TIMESPAN="$DEF_TIMESPAN"
tstrt=$((tend-$DEF_TIMESPANSECS))
TIMESTART=$(dispdate "+%Y-%m-%d+%H:%M:%S" -t $tstrt)
# Compute time span if start time provided.
elif [ -n "$TIMESTART" ]; then
tstrt=$(dispdate "$TIMESTART" -t)
TIMESPAN=$((tend - $tstrt))
if [ "$TIMESPAN" -le 0 ]; then
TIMESPAN="$DEF_TIMESPAN"
else
TIMESPAN=$(echo "($TIMESPAN + 3599) / 3600" | bc)"hours"
fi
fi
TIMESTARTARG=$TIMESTART
# Extract poll ending time elements.
YEAR=$(dispdate "$TIMEEND" "+%Y")
MONTH=$(dispdate "$TIMEEND" "+%m")
DAY=$(dispdate "$TIMEEND" "+%d")
HOUR=$(dispdate "$TIMEEND" "+%H")
translate_wind_direction_heading() {
heading=$1
if [ "$heading" = "N" ]; then wd="0"
elif [ "$heading" = "NNE" ]; then wd="22.5"
elif [ "$heading" = "NE" ]; then wd="45"
elif [ "$heading" = "ENE" ]; then wd="67.5"
elif [ "$heading" = "E" ]; then wd="90"
elif [ "$heading" = "ESE" ]; then wd="112.5"
elif [ "$heading" = "SE" ]; then wd="135"
elif [ "$heading" = "SSE" ]; then wd="157.5"
elif [ "$heading" = "S" ]; then wd="180"
elif [ "$heading" = "SSW" ]; then wd="202.5"
elif [ "$heading" = "SW" ]; then wd="225"
elif [ "$heading" = "WSW" ]; then wd="247.5"
elif [ "$heading" = "W" ]; then wd="270"
elif [ "$heading" = "WNW" ]; then wd="292.5"
elif [ "$heading" = "NW" ]; then wd="315"
elif [ "$heading" = "NNW" ]; then wd="337.5"
else wd="$heading"; fi
echo $wd
}
process_noaa_raws_data () {
# Station numid passed as first argument.
STAID=$1
# Poll start time passed as second argument.
# Do not file data reports not newer than this time.
TIMESTART="$2"
if [ -z "$TIMESTART" ]; then tstrt=0;
else tstart=$(dispdate "$TIMESTART" -t); fi
# Poll end time passed as third argument.
# Data lines contain time only so prefix time with poll ending time.
# Data line time is greater than poll ending time then roll back 1 day.
TIMEEND="$3"
date=$(dispdate "$TIMEEND" "+%m/%d/%Y")
tend=$(dispdate "$TIMEEND" -t)
tlast=$tend
# Translate file returned stripping out HTML, trim spaces, remove blank lines.
founddatatable=""
founddataline=""
processdataline="false"
cat $OFILE | \
# Put a space in data table following
for data parsing.
sed -e 's/ | / | /g' | \
# Remove all HTML tags.
sed -n '/^$/!{s/<[^>]*>//g;p;}' | \
# Translate TABs and $nbsp to spaces.
# Translate ° to deg.
# Compress multiple spaces to one space.
# Remove spaces at start of lines.
# Empty lines filled with spaces only.
# Remove blank lines.
sed -e 's/\t/ /g' \
-e 's/ / /g' \
-e 's/ / /g' \
-e 's/°/deg/g' \
-e 's/°/deg/g' \
-e 's/ */ /g' \
-e 's/^ *//' \
-e 's/^ *$//g' \
-e '/^$/d' | \
# Process trimmed output looking for data lines.
while read line; do
# Data header starts with string: 'Tabular Listing:'
if [ -z "$founddatatable" ]; then
founddatatable=$(echo $line | grep "Tabular Listing:")
fi
if [ -n "$founddatatable" ]; then
# Data readout lines start with the time as 'HH:MM '
founddataline=$(echo $line | grep "^[0-9]*:[0-9][0-9] ")
if [ -n "$founddataline" ]; then
processdataline="true"
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
# Translate data field space separators into newlines for parsing.
datalist=$(echo "$line" | sed -e 's/ */\n/')
# Encode data reports for filing.
# First field is time.
cnt=0
rm -f $TMPFILE
for data in $datalist; do
# First field is time.
if [ $cnt -eq 0 ]; then
# Adjust time for rollover to previous day.
time="$data:00"
datetime=$date"-"$time
trpt=$(dispdate $datetime -t)
if [ "$trpt" -gt "$tlast" ]; then
trpt=$((trpt-86400))
date=$(dispdate "+%m/%d/%Y" -t $trpt)
datetime=$date"-"$time
fi
tlast=$trpt
# Quit data filing if data report time before start time.
if [ "$trpt" -lt "$tstart" ]; then break; fi
else
# Translate quality flag.
if [ "$data" = "OK" ]; then data="1"; fi
# Translate wind heading to direction in degrees.
data=$(translate_wind_direction_heading $data)
echo $REMOTETAG $cnt $data $datetime >>$TMPFILE
fi
cnt=$((cnt+1))
done
# File data buffered in temporary file
if [ -s "$TMPFILE" ]; then
# File data reports unless disabled.
if [ "$nofile" = "false" ]; then
$FILEDATACMD <$TMPFILE
fi
fi
# Quit data filing if only lastest report filed
if [ $polltype -lt 2 ]; then break; fi
# Quit data filing if data report time before start time.
if [ "$trpt" -lt "$tstart" ]; then break; fi
else
if [ "$processdataline" = "false" ]; then
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
fi
fi
fi
done
}
process_cdec_raws_data () {
# Station numid passed as first argument.
STAID=$1
# Poll start time passed as second argument.
# Do not file data reports not newer than this time.
TIMESTART="$2"
if [ -z "$TIMESTART" ]; then tstrt=0;
else tstart=$(dispdate "$TIMESTART" -t); fi
# Poll end time passed as third argument.
# Data lines contain time only so prefix time with poll ending time.
# Data line time is greater than poll ending time then roll back 1 day.
TIMEEND="$3"
date=$(dispdate "$TIMEEND" "+%m/%d/%Y")
tend=$(dispdate "$TIMEEND" -t)
tlast=$tend
# Translate file returned stripping out HTML, trim spaces, remove blank lines.
founddatatable=""
founddataline=""
processdataline="false"
rm -f $TMPFILE
cat $OFILE | \
# Put a space in data table following | for data parsing.
sed -e 's/ | / | /g' | \
# Remove all HTML tags.
sed -n '/^$/!{s/<[^>]*>//g;p;}' | \
# Translate TABs and $nbsp to spaces.
# Translate ° to deg.
# Compress multiple spaces to one space.
# Remove spaces at start of lines.
# Empty lines filled with spaces only.
# Remove blank lines.
sed -e 's/\t/ /g' \
-e 's/ / /g' \
-e 's/ / /g' \
-e 's/°/deg/g' \
-e 's/°/deg/g' \
-e 's/ */ /g' \
-e 's/^ *//' \
-e 's/^ *$//g' \
-e '/^$/d' | \
# Process trimmed output looking for data lines.
while read line; do
echo "line: $line"
# Data header starts with string: 'Date / Time'
if [ -z "$founddatatable" ]; then
founddatatable=$(echo $line | grep "Date / Time")
fi
if [ -n "$founddatatable" ]; then
echo "found data table"
# Data readout lines start with the date as 'MM/DD/YYYY '
founddataline=$(echo $line | grep "^[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] ")
if [ -n "$founddataline" ]; then
processdataline="true"
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
# Translate data field space separators into newlines for parsing.
echo "line before datalist: $line"
datalist=$(echo "$line" | sed -e 's/ */\n/')
# Encode data reports for filing.
# First field is date, second field is time.
cnt=-1
for data in $datalist; do
# First field is date.
echo "cnt= $cnt"
if [ $cnt -eq -1 ]; then
date="$data"
elif [ $cnt -eq 0 ]; then
time="$data"
datetime=$date"-"$time
trpt=$(dispdate $datetime -t)
echo "trpt= $trpt"
echo "tstart= $tstart"
if [ "$trpt" -lt "$tstart" ]; then break; fi
elif [ $data != "$MISSING" ]; then
echo $REMOTETAG $cnt $data $datetime >>$TMPFILE
echo "Submit value: $REMOTETAG $cnt $data $datetime"
fi
cnt=$((cnt+1))
done
else
if [ "$processdataline" = "false" ]; then
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
fi
fi
fi
done
# File data buffered in temporary file
if [ -s "$TMPFILE" ]; then
# File data reports unless disabled.
if [ "$nofile" = "false" ]; then
$FILEDATACMD <$TMPFILE
fi
fi
}
process_mesowest_raws_data () {
# Station numid passed as first argument.
STAID=$1
# Poll start time passed as second argument.
# Do not file data reports not newer than this time.
TIMESTART="$2"
echo "TIMESTART: $TIMESTART"
if [ -z "$TIMESTART" ]; then tstrt=0;
else tstart=$(dispdate "$TIMESTART" -t); fi
# Poll end time passed as third argument.
# Data lines contain time only so prefix time with poll ending time.
# Data line time is greater than poll ending time then roll back 1 day.
TIMEEND="$3"
date=$(dispdate "$TIMEEND" "+%m/%d/%Y")
tend=$(dispdate "$TIMEEND" -t)
tlast=$tend
# Get the current time for later comparison
currenttime=`date '+%H%M'`
previousdate="false"
# Translate file returned stripping out HTML, trim spaces, remove blank lines.
founddatatable=""
founddataline=""
processdataline="false"
rm -f $TMPFILE
echo "OFILE= $OFILE"
# Replace blank table cells with a placeholder, retain tags for now.
# This is to accommodate blank cells in the input table where there is missing data.
# could not find a way to do this with sed...
perl -i -p0e 's/ | \s*?<\/td>/ | -999<\/td>/g' $OFILE
cat $OFILE | \
# Put a space in data table following | for data parsing.
sed -e 's/ | / | /g' | \
# Change to ENDTABLE for data parsing
# sed -e 's//ENDTABLE/g' | \
# Remove all HTML tags.
sed -n '/^$/!{s/<[^>]*>//g;p;}' | \
# Remove the lines associated with ]*//g' | \
sed -e 's/target="_blank">//g' | \
# Translate TABs and $nbsp to spaces.
# Translate ° to deg.
# Compress multiple spaces to one space.
# Remove spaces at start of lines.
# Empty lines filled with spaces only.
# Remove blank lines.
sed -e 's/\t/ /g' \
-e 's/ / /g' \
-e 's/ / /g' \
-e 's/°/deg/g' \
-e 's/°/deg/g' \
-e 's/ */ /g' \
-e 's/^ *//' \
-e 's/^ *$//g' \
-e '/^$/d' | \
# Process trimmed output looking for data lines.
while read line; do
# Data header starts with string: 'Time'
if [ -z "$founddatatable" ]; then
founddatatable=$(echo $line | grep "Time")
fi
if [ -n "$founddatatable" ]; then
# check for ENDTABLE and end loop if so
foundendtable=$(echo $line | grep "ENDTABLE")
if [ "$foundendtable" = "true" ]; then
break;
fi
# Data readout lines start with the time as 'hh:mm'
foundtime=$(echo $line | grep "^[0-9]*:[0-9][0-9]")
data=$(echo "$line" | sed -e 's/ */\n/')
if [ -n "$foundtime" ]; then
processdataline="true"
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
# Translate data field space separators into newlines for parsing.
# Encode data reports for filing.
# First field is time.
cnt=0
# need to get date correct
date=`date '+%m/%d/%Y'`
time=$data
datatime=$data
datatime=`echo $datatime | sed 's/://g'`
if [ "$previousdate" = "false" ]; then
if [ "$datatime" -gt "$currenttime" ]; then
previousdate="true"
echo "Setting date to previous date!!!!! (datatime: $datatime, currenttime: $currenttime)"
fi
fi
if [ "$previousdate" = "true" ]; then
date=`date '+%m/%d/%Y' --date='yesterday'`
fi
datetime=$date"-"$time
trpt=$(dispdate $datetime -t)
echo "Comparing trpt: $trpt to tstart: $tstart"
if [ "$trpt" -lt "$tstart" ]; then
break;
fi
elif [ "$processdataline" = "true" ] && [ $data != "$MISSING" ]; then
cnt=$((cnt+1))
alphanumeric=$(echo $data | grep "[a-zA-Z]*")
if [ "$alphanumeric" ]; then
# Translate quality flag.
if [ "$data" = "OK" ]; then
data="1"
else
# Translate wind heading to direction in degrees.
data=$(translate_wind_direction_heading $data)
fi
fi
echo $REMOTETAG $cnt $data $datetime >>$TMPFILE
echo "Sending to file: $REMOTETAG $cnt $data $datetime"
else
if [ "$processdataline" = "false" ]; then
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then echo "$line"; fi
fi
fi
fi
fi
done
# File data buffered in temporary file
if [ -s "$TMPFILE" ]; then
# File data reports unless disabled.
if [ "$nofile" = "false" ]; then
$FILEDATACMD <$TMPFILE
echo "FILEDATACMD: $FILEDATACMD"
fi
fi
}
# Append command output to log file
(
if [ "$quiet" = "false" ]; then
echo "$(nstime) $PGNAME: $PGDESC started..."
if [ "$debug" -gt 0 ]; then echo "$COMMANDLINE"; fi
fi
# Check if process locked.
locked="false"
if [ "$force" = "false" ]; then
if [ -f $LOCKFILE ]; then
echo "$(nstime) $PGNAME: $PGDESC locked!"
locked="true"
fi
fi
# Execute command if not locked.
if [ "$locked" = "false" ]; then
echo "$(nstime) $COMMANDLINE" >$LOCKFILE
# Poll all stations in list.
echo "$STAIDS" | sed -e "s/,/\n/g" | \
while read STAID; do
# Get station name, last polled time, timeout, remote tag, skip if not defined.
STANAME=$(echo "select name from station where numid=$STAID;" | $PSQL -AtF,)
STALASTTIMEPOLLED=$(echo "select last_time_polled from station where numid=$STAID;" | $PSQL -AtF,)
TIMEOUT=$(echo "select connection_timeout from station where numid=$STAID;" | $PSQL -AtF,)
REMOTETAG=$(echo "select remote_tag from station where numid=$STAID;" | $PSQL -AtF,)
if [ "$quiet" = "false" ]; then
echo "Poll Station ID: $STAID $STANAME"
fi
if [ -z "$REMOTETAG" ]; then
if [ "$quiet" = "false" ]; then
echo "Poll Station ID: $STAID $STANAME - ERROR: No remote station tag defined."
fi
continue
fi
# Set default timeout
TIMEOUT=${TIMEOUT:-$DEF_TIMEOUT};
if [ "$TIMEOUT" = "0" ]; then TIMEOUT=$DEF_TIMEOUT; fi
# Set start time to last time polled if undefined.
if [ -z "$TIMESTARTARG" ]; then
TIMESTART="$STALASTTIMEPOLLED"
# Compute time span from ending time to last time polled.
tstrt=$(dispdate "$TIMESTART" -t)
TIMESPAN=$((tend - $tstrt))
if [ "$TIMESPAN" -le 0 ]; then
TIMESPAN="$DEF_TIMESPAN"
else
TIMESPAN=$(echo "($TIMESPAN + 3599) / 3600" | bc)"hours"
fi
fi
# Get poll line to use as data filing source, default to 1.
SOURCE=$(echo "select line from station where numid=$STAID;" | $PSQL -AtF,)
SOURCE=${SOURCE:-1}
# Get web page URL for station, skip if not defined.
# Translate URL to replace REMOTETAG, TIMEEND, and TIMESPAN.
URL=$(echo "select comm_path from station where numid=$STAID;" | $PSQL -AtF, | \
sed \
-e "s/_REMOTETAG_/$REMOTETAG/" \
-e "s/_TIMEEND_/$TIMEEND/" \
-e "s/_DAY_/$DAY/" \
-e "s/_MONTH_/$MONTH/" \
-e "s/_YEAR_/$YEAR/" \
-e "s/_HOUR_/$HOUR/" \
-e "s/_TIMESPAN_/$TIMESPAN/")
if [ -z "$URL" ]; then
if [ "$quiet" = "false" ]; then
echo "Poll Station ID: $STAID $STANAME - ERROR: No URL defined."
fi
continue
fi
# Set data process type
PROCESSCDECRAWSDATA="false"
if [ -n "$(echo $URL | grep cdec)" ]; then PROCESSCDECRAWSDATA="true"; fi
PROCESSNOAARAWSDATA="false"
if [ -n "$(echo $URL | grep noaa)" ]; then PROCESSNOAARAWSDATA="true"; fi
PROCESSMESOWESTRAWSDATA="false"
if [ -n "$(echo $URL | grep mesowest)" ]; then PROCESSMESOWESTRAWSDATA="true"; fi
# Make data filing command.
# Display data filed if not quiet.
FILEDATACMD="starpt -l2 -i -S$SOURCE"
if [ "$quiet" = "false" ]; then FILEDATACMD=$FILEDATACMD" -d$debug";
else FILEDATACMD=$FILEDATACMD" -q"; fi
# Get web page from URL for processing.
if [ "$quiet" = "false" ]; then
if [ "$debug" -gt 0 ]; then
echo wget --no-check-certificate -q -O $OFILE $URL
fi
fi
rm -f $OFILE
timeout $TIMEOUT wget --no-check-certificate -q -O $OFILE $URL
# Process web page buffer if not empty.
if [ -s $OFILE ]; then
# Process CDEC RAWS data
if [ "$PROCESSCDECRAWSDATA" = "true" ]; then process_cdec_raws_data $STAID "$TIMESTART" "$TIMEEND"; fi
if [ "$PROCESSNOAARAWSDATA" = "true" ]; then process_noaa_raws_data $STAID "$TIMESTART" "$TIMEEND"; fi
if [ "$PROCESSMESOWESTRAWSDATA" = "true" ]; then process_mesowest_raws_data $STAID "$TIMESTART" "$TIMEEND"; fi
# Update station last time polled.
args="-qAtF,"
if [ "$debug" -gt 0 ]; then args="-aAtF,"; fi
$PSQL $args << _EOF_
UPDATE station SET last_time_polled='$timestamp' WHERE numid
IN ($STAIDS) AND
(last_time_polled IS null OR last_time_polled<'$timestamp');
_EOF_
fi
done
# Clean up
rm -rf $OFILE
rm -rf $TMPFILE
# Unlock
rm -rf $LOCKFILE
fi
# Append command output to log file
if [ "$quiet" = "false" ]; then
echo "$(nstime) $PGNAME: $PGDESC finished."
fi
) | tee -a $LOGFILE
|