#!/usr/bin/perl

#This file is the original file(csv generation) with support for html view 

##################################################################################################

#usage: perl dods_filter_column.pl 'http://nccoos.unc.edu/cgi-bin/nph-dods/data/nws_metar/proc_data/latest_v2.0/nws-K11J-metar-latest.nc.ascii?wspd' 'seconds since 1970-1-1 00:00:00' 12345

#This perl program can be run stand-alone with the correct parameters and/or in conjunction with a sample calling php webpage - dods_filter_column.php (as a web service) which supplies the parameters from the URL session parameters.

##stand alone
#Run stand alone it can perform row-to-column transformation for other programs which need the data in a CSV(Comma Separated Value) format

#Run stand alone it can also serve as a base template for further modification to create relational database population files with additional row output tailoring for the specific INSERT or COPY file table need.

#Note that the NCO ( http://nco.sourceforge.net ) utility operator 'ncrcat' can also be used to concatenate similar files along the time axis to create a larger composite netCDF file which might be easier to query or transform.

##web service
#Run as called from a web service(see sample calling php webpage - dods_filter_column.php ) it can provide the same transformation within the browser clients (without the need for installing software and possibly as part of other pipelined web service processes) 

#If running locally change the $tmp_dir reference as needed

################### 
#The code below uses the UDUNITS perl package for transforming netCDF times to ISO type datetimes.  To install this package on a Redhat Linux server I had to perform the below steps(see the INSTALL document, talks about other prerequisites like perl 5.x, etc) as root:

#download and unzip version 1.12.2 from Unidata(also available at http://carocoops.org/resources/netcdf/udunits-1.12.2.tar.Z ) 

#add the following 3 lines to the /src/configure file to recognize a Linux system
#	CC=gcc
#	CFLAGS=-Df2cFortran
#	FC=g77

#run ./configure from /udunits-1.12.2/src 
#run perl Makefile.PL from /udunits-1.12.2/src/perl
#run make test from /udunits-1.12.2/src/perl
#run make install from /udunits-1.12.2/src/perl
###################

##netCDF file assumptions
#The code assumes that the time values line will be the third line listed and there are a corresponding number of elements for each of the datatypes included in the query.


##################################################################################################

use strict;

my $tmp_dir = '/tmp/ms_tmp';

my $dods_url = shift;
my $units_value = shift;
my $process_id = shift;
my $submit_type = shift;
my $qc_source = shift;

#call data args into scope from xml document

use LWP::Simple;
my $content;

$content = get $qc_source;
die "Couldn't get $qc_source" unless defined $content;

use XML::Simple;
my $xs1 = XML::Simple->new();
my $doc = $xs1->XMLin($content, forcearray => 1);

=comment
#to view the contents of the $doc array
use Data::Dumper;
print Dumper($doc);
=cut

my $sst_range_annual_high = $doc->{'observation_types'}->[0]->{'sea_surface_temperature'}->[0]->{'range_test'}->[0]->{'annual'}->[0]->{'high'}->[0] ;
my $sst_range_annual_low = $doc->{'observation_types'}->[0]->{'sea_surface_temperature'}->[0]->{'range_test'}->[0]->{'annual'}->[0]->{'low'}->[0] ;


my $csv_filename;
if ($submit_type eq 'download') {
	$csv_filename = "cc_data_$process_id.csv";
}
else { #view
	$csv_filename = "cc_data_$process_id.html";
}

open (OUT_FILE, ">$tmp_dir/$csv_filename");


use UDUNITS;

# udunits initialization
UDUNITS::init("/usr2/home/jcothran/udunits-1.12.2/src/lib/udunits.dat") == 0 
  || die "ABORT! Cannot initialize udunits.\n";

my $dods_input_filename = "dods_input_$process_id.txt";

`/usr/bin/wget -O $tmp_dir/$dods_input_filename $dods_url`;

open (DODS_INPUT, "$tmp_dir/$dods_input_filename");

my @time_values;

my $num_obs_type = 0;
my @obs_values;

my $line;
my $line_count = 0;

while ($line = <DODS_INPUT>) {
	$line_count++;

	#drop first dataset name line
        if ($line_count == 1) { next; }

	#get time elements
        if ($line_count == 2) {
                @time_values = split(/\,/,$line);
		next;
	}

        if ($line_count % 2) {
		$num_obs_type++;
		push (@obs_values, split(/\,/,$line));
		next;
	}

}

close (DODS_INPUT);

my $base_time = UDUNITS::scan($units_value)
    || die "ABORT! Error with time units.\n";
$base_time->istime()
    || die "ABORT! Invalid units for time.\n";

my ($i,$j);
my ($base_year, $base_month, $base_day, $base_hour, $base_minute, $base_second);
my $time_formatted_value;
my $obs_title_formatted;

#print header line

if ($submit_type eq 'download') {
	print OUT_FILE "time";
}
else { #view
	print OUT_FILE "<table><tr><td>time</td>";
}

for ($j=0; $j < $num_obs_type; $j++) {
	$obs_title_formatted = @obs_values[$j*scalar(@time_values)];
	#reduce title.title to just title
	$obs_title_formatted =~ s/\.(.*)// ;

	if ($submit_type eq 'download') {
		print OUT_FILE ",".$obs_title_formatted.",qc flag";
	}
	else { #view
		print OUT_FILE "<td>".$obs_title_formatted."</td><td>qc flag</td>";
	}

}

if ($submit_type eq 'download') {
	print OUT_FILE "\n";
}
else { #view
        print OUT_FILE "</tr>";
}

#print data
for ($i=1; $i < scalar(@time_values); $i++) 
{
  
    # convert the time value to new value based on the time units
    $base_time->valtocal($time_values[$i], $base_year, $base_month, $base_day, $base_hour, $base_minute, $base_second) == 0
      || die "ABORT! Invalid units for time.\n";
    $time_formatted_value = $base_year.'-'
      .sprintf("%02d",$base_month).'-'
      .sprintf("%02d",$base_day).' '
      .sprintf("%02d",$base_hour).':'
      .sprintf("%02d",$base_minute).':'
      .sprintf("%02d",$base_second);

        if ($submit_type eq 'download') {
		print OUT_FILE $time_formatted_value;
        }
        else { #view
		print OUT_FILE "<td>".$time_formatted_value."</td>";
        }

	for ($j=0; $j < $num_obs_type; $j++) {
		if ($i == scalar(@time_values)-1) {
			chomp(@obs_values[$i+($j*scalar(@time_values))]);	
		}

		my $this_obs = @obs_values[$i+($j*scalar(@time_values))];
		my $qc_flag;

		if (($this_obs > $sst_range_annual_high) || ($this_obs < $sst_range_annual_low)) { $qc_flag = 'r 2'; }
		else { $qc_flag = 'R'; }

	        if ($submit_type eq 'download') {
			print OUT_FILE ",$this_obs,$qc_flag";
        	}
        	else { #view
			print OUT_FILE "<td>$this_obs</td><td>$qc_flag</td>";
        	}

	}

	if ($submit_type eq 'download') {
        	print OUT_FILE "\n";
	}
	else { #view
        	print OUT_FILE "</tr>";
	}

}

if ($submit_type ne 'download') {
	print OUT_FILE "</table>";
}


close (OUT_FILE);

`rm $tmp_dir/$dods_input_filename`;

exit 0;