Shell Script for Directing GOES Data Ingest

#!/bin/bash


#
# All computations are done here.
function do_it {

	ppid=`ps -f $$ | tail +2 | awk '{print  $3}'`
	echo ----------------------------------------------------------
	echo ----------------------------------------------------------
	echo - Starting GOES ${current_buoy} $$ at `date`
	echo - PID is $$, PPID=${ppid}
	echo - `hostname -a`
	echo
	
	#
	# Use the date for filename conventions
	year=`date +%Y`
	month=`date +%m`
	day=`date +%d`
	hour=`date +%H`
	minute=`date +%M`
	second="00"
	partial_fname=${year}_${month}_${day}_${hour}
	
	
	mooring_id=${current_buoy}
	
	
	#
	# Now grab the platform id.  
	# Maybe it's kind of wasteful to do this in two steps, but it's more
	# in line with traditional shell scripting to NOT treat input like line-by-line
	query='select goes_platform_id from mooring where mooring_id="'${mooring_id}'"'
	goes_platform_id=`echo $query | mysql -h micmac.umeoce.maine.edu -u webuser -pwww gomoos_control | tail +2`
	
	if [ ${goes_platform_id} = "NULL" ]; then
		
		#
		echo No GOES platform id for mooring ${mooring_id}
		exit 1
	
	fi
	
	echo ${goes_platform_id} is paired with ${mooring_id}
	
	
	#
	# Ok, now download the goes file for this mooring
	# Place it in a temporary directory
	download_dir=${DATA_ROOT}/incoming/goes/$$
	mkdir -p $download_dir
	cd $download_dir
	echo Downloading now into `pwd`

	download_file=${mooring_id}_${partial_fname}.dat
	
	retrieve_goes_buffer.pl ${goes_platform_id} ${download_file} > /dev/null
	
	echo Data retrieved into ${download_file} 
	
	#
	# Did the download actually happen?
	if [ -e ${download_file} ] ; then
	
		#
		# invoke matlab to process the data
		export MOORING_ID=${mooring_id}
		nice matlab -nojvm -nosplash -r  process_goes_wrapper 
	
		#
		# copy the files to a more permanent location
		mkdir -p ${DATA_ROOT}/incoming/goes/${mooring_id}/${year}/${month}/${day}
		mv ${download_file} ${DATA_ROOT}/incoming/goes/${mooring_id}/${year}/${month}/${day}
	
		sh ${RUNTIME_ROOT}/bin/cache_updated_webpages.sh ${mooring_id} 
		sh ${RUNTIME_ROOT}/bin/cache_table_js_pages.sh ${mooring_id} 
	
		echo starting rsync transfer at `date`
		export RSYNC_PASSWORD=gomoos_rsync_transfer
		rsync --verbose  --progress --stats --compress  \
		      --recursive --owner --times --perms --links \
			   --exclude "*~" \
			  /data/gomoos/buoy/archive/${current_buoy}  gomoos@gyre::grbt
		echo apparently finished with rsync transfer at `date`
	fi
	
	#
	# Remove the temporary download directory now that we are done with it.
	cd ..
	rm -rf $download_dir
	
	

	echo - Ending GOES ${current_buoy} $$ at `date`
	echo - PID is $$ 
	echo ----------------------------------------------------------
	echo ----------------------------------------------------------
	echo
}


#
# How many input arguments?  If two, then
# the 2nd argument is the "project".  Set this to "blue_hill_bay", for example.
# If not supplied, "gomoos" is the default.
echo Script invoked as "$0: $@"
if [ $# -eq 2 ]; then
	project=${2}
else
	project=gomoos
fi


export DATA_ROOT=/data/${project}/buoy
export RUNTIME_ROOT=/data/gomoos/buoy
export BUOY_PROCESSING_CALLING_ENVIRONMENT="shell script"
export BUOY_PROJECT=${project}
incoming_directory=$DATA_ROOT/incoming


#
# Set up the rest of the environment.
. ${RUNTIME_ROOT}/bin/setup_gomoos_environment.sh



current_buoy=${1}



#
# check the pid file.  If it exists, then don't run
pidfile=${RUNTIME_ROOT}/run/${current_buoy}_goes
if ls ${pidfile}
then
	#
	# pid file exists. must already be running?
#	echo "Pid file ${pidfile} exists, too much going on.  Remove this file if in error." | mail jevans
	exit 1
fi

touch ${pidfile}

master_log_file=${DATA_ROOT}/log/buoy.log

do_it >> ${pidfile}
#do_it 


cat ${pidfile} >> ${master_log_file}


#
# finish by removing the pid file
echo removing pid file ${pidfile}
rm -f ${pidfile}