[Wrf-users] Re: Input data for forecasting
Graziano Giuliani
giuliani at lamma.rete.toscana.it
Fri Feb 3 02:21:48 MST 2006
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
I use something more sofisticated on a Linux system, but the concept is
the same...
Cheers,
Graziano.
#########################################################################
#!/bin/bash
############################################################################
#
# Project : LaMMA Forecast System
# Author : Graziano Giuliani (LaMMA Regione Toscana)
# Date : Thu Dec 29 10:38:27 UTC 2006
#
############################################################################
#
# Concept is this: Try the ftp2me service on more server more times until
# data are on disk or a timeout occurs.
#
export TZ=GMT
export LANG=C
ftp2me ( )
{
if [ $# -lt 5 ]
then
echo "Not enough arguments...."
echo "Usage:"
echo " $0 dest_dir start span stride base_url [moreurls]"
echo
echo "Example:"
echo
echo "$0 /gfsdata 00 72 6 http://www.ncep.noaa.gov/cgi-bin/ftp2u_avn_dir.sh"
echo
return 255
fi
DOWNDIR=$1
RUNSTART=$2
ENDH=$3
INTERVAL=$4
shift 4
NURLS=$#
declare BASEURL[$NURLS]
BASEURL=($@)
BASEBASE=gfs
BASEFILE=$BASEBASE.t`printf "%02d" $RUNSTART`z.master.grbf
BASEMODEL=gfs
BASEDATE=`date +"%Y%m%d" --date "today 00:00:00.0000+00:00"`
STARTH=0
LEVELS="1000_mb 925_mb 850_mb 700_mb 500_mb 400_mb 300_mb 250_mb \
200_mb 150_mb 100_mb 70_mb 50_mb 30_mb 20_mb 10_mb \
MSL sfc 2_m_above_gnd 10_m_above_gnd 0-10_cm_down \
10-40_cm_down 40-100_cm_down 100-200_cm_down"
VARIABLES="HGT RH TMP UGRD VGRD PRMSL SOILW LAND ICEC WEASD PRES"
# DOMINIO PICCOLO
LEFTLON=-30
RIGHTLON=40
TOPLAT=70
BOTTOMLAT=20
TIMESAVE=3hr
TRYFORURL=3
TIMESLEEP=300
############################################################################
#
# End configurable parameters
#
############################################################################
URLSTRING="?"
for (( IFILE = STARTH; IFILE <= ENDH; IFILE = IFILE + INTERVAL ))
do
GFILE="$BASEFILE`printf \"%02d\" $IFILE`"
URLSTRING="${URLSTRING}file=$GFILE&"
done
URLSTRING="${URLSTRING}wildcard=&"
for LEVEL in $LEVELS
do
URLSTRING="${URLSTRING}lev_${LEVEL}=on&"
done
for VAR in $VARIABLES
do
URLSTRING="${URLSTRING}var_${VAR}=on&"
done
URLSTRING="${URLSTRING}subregion=&"
URLSTRING="${URLSTRING}leftlon=${LEFTLON}&rightlon=${RIGHTLON}&"
URLSTRING="${URLSTRING}toplat=${TOPLAT}&bottomlat=${BOTTOMLAT}&"
URLSTRING="${URLSTRING}results=SAVE&rtime=${TIMESAVE}&"
URLSTRING="${URLSTRING}dir=%2F${BASEMODEL}${BASEDATE}"
#URLSTRING="${URLSTRING}dir=%2Frotating"
echo "Starting retrieval phase at `date`"
RETRY=0
RESULT=255
while [ $RETRY -lt $NURLS -a $RESULT -ne 0 ]
do
for (( URLTRY = 0; URLTRY < TRYFORURL; URLTRY++ ))
do
QUERYSTRING=${BASEURL[$RETRY]}$URLSTRING
echo "Sending query to ${BASEURL[$RETRY]} ($(( $URLTRY + 1 )))....."
wget -O result.html -o /dev/null ${QUERYSTRING}
echo "Query sent."
EGREP_OPTS='-is'
EGREP_PATT='Results were saved in'
EGREP_FILTER="s+^.*href=\"[ ]*\([^ \"]*\)[ ]*\".*+\1+"
SEDMAGIC='p'
RESURL=`egrep $EGREP_OPTS $EGREP_PATT result.html | \
sed "$EGREP_FILTER" | sed -n "$SEDMAGIC"`
touch $DOWNDIR/lock
if [ ! "X$RESURL" = "X" ]
then
wget -c --no-remove-listing -o /dev/null -P $DOWNDIR $RESURL/$BASEFILE*
if [ $? -ne 0 ]
then
echo "Error in file retrieval."
echo "Sleeping for $TIMESLEEP seconds."
sleep $TIMESLEEP
RESULT=255
else
cat $DOWNDIR/.listing
rm -f $DOWNDIR/.listing
echo "Successful file retrieval."
RESULT=0
rm -f result.html
rm -f $DOWNDIR/lock
break
fi
else
echo "Error in file retrieval."
echo "Sleeping for $TIMESLEEP seconds."
sleep $TIMESLEEP
RESULT=255
fi
rm -f result.html
done
RETRY=$(( $RETRY + 1 ))
if [ $RESULT -ne 0 -a $RETRY -lt $NURLS ]
then
echo
echo "Switching to next URL."
echo
fi
done
echo "End retrieval phase at `date`"
return $RESULT
}
############################################################################
# Processing phase
############################################################################
if [ $# -ne 3 ]
then
echo
echo "Wrong arguments."
echo
echo "I need runstart runlen runstep, as in"
echo
echo " "`basename $0`" 00 72 06"
echo
exit 1
fi
# if needed in operative, save output to a log file and work silently
#hh=`date +"%Y_%m_%d_%H_%M_%S"`
#exec 1>$HOME/log/`basename $0`_$hh.log
#exec 2>&1
run=$1
runlen=$2
runstep=$3
# How much to wait before giving up ? Set to 6 hours.
waitsecs=21600
# Calculate total number of files to retrieve
nfiles=$(( $runlen / $runstep + 1 ))
# We are operative, so we want data for today !
# rundate=2006020100
rundate=`date +"%Y%m%d"$run`
rundir=$HOME/run${run}
dir_arc=$HOME/archive
[ -d $rundir ] || mkdir -p $rundir
[ -d $dir_arc ] || mkdir -p $dir_arc
rm -f ${rundir}/gfs*
# First try
ftp2me ${rundir} ${run} ${runlen} ${runstep} \
http://nomads6.ncdc.noaa.gov/cgi-bin/ftp2u_gfs0.5.sh \
http://nomad3.ncep.noaa.gov/cgi-bin/ftp2u_gfs0.5.sh \
http://nomad5.ncep.noaa.gov/cgi-bin/ftp2u_gfs_master.sh
now=`date +%s`
stopwait=$(( $now + $waitsecs ))
num_grib=0
while [ $now -lt $stopwait ]
do
num_grib=`ls -lrt ${rundir}/gfs* 2>/dev/null | wc -l`
if [ $num_grib -ne $nfiles ]
then
# The try has not been successful..
# Try again.
echo "Sleep 3 minutes and retry...."
sleep 180
ftp2me ${rundir} ${run} ${runlen} ${runstep} \
http://nomads6.ncdc.noaa.gov/cgi-bin/ftp2u_gfs0.5.sh \
http://nomad3.ncep.noaa.gov/cgi-bin/ftp2u_gfs0.5.sh \
http://nomad5.ncep.noaa.gov/cgi-bin/ftp2u_gfs_master.sh
else
# Keep a copy of data
cd ${rundir}
/bin/tar -czf $dir_arc/${rundate}_gfs_05.tar.gz gfs.t${run}z.master*
echo "Suceess !"
break
fi
now=`date +%s`
done
# We do not have data...
if [ $num_grib -ne $nfiles ]
then
echo "Not all files found, sorry...."
echo "I have only $num_grib out of needed $nfiles"
ls -lrt ${rundir}/gfs*
exit 1
fi
exit 0
- --
\ | /
(@ @)
-------------------------o00-(_)-00o-----------------------------
LaMMA - Laboratorio per la Meteorologia e la Modellistica Ambientale
Laboratory for Meteorology and Environmental Modelling
Via Madonna del Piano, 50019 Sesto Fiorentino (FI)
tel: + 39 055 4483049
fax: + 39 055 444083
web: www.lamma.rete.toscana.it
e-mail: giuliani at lamma.rete.toscana.it
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.2 (GNU/Linux)
iD8DBQFD4yCwdWEgLYqZ9NERAv+LAJwJj2iCCRzmAqcqHLxLjBJMowXQoQCeMUEP
m9ZoBvNdPJ/ZC7Y0u02UpbM=
=2LHP
-----END PGP SIGNATURE-----
More information about the Wrf-users
mailing list