/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package m.climate;
import csip.ServiceException;
import csip.SessionLogger;
import gisobjects.GISObject;
import gisobjects.GISObjectException;
import gisobjects.GISObjectFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import ucar.ma2.Array;
import ucar.ma2.ArrayFloat;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import ucar.nc2.dt.GridCoordSystem;
import ucar.nc2.dt.GridDatatype;
import ucar.nc2.dt.grid.GridDataset;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.TimeDuration;
/**
*
* @author pattersd
*/
public class ThreddsModelDataService extends ClimateModelDataService {
public SessionLogger logger;
public void setLogger(SessionLogger logger) {
this.logger = logger;
}
public JSONArray extractFromThredds(List<String> ncFiles,
JSONObject inputZoneFeatures,
String startDateStr, String endDateStr, String ts)
throws ServiceException, JSONException, ParseException, IOException, InvalidRangeException, GISObjectException {
SimpleDateFormat ft = new SimpleDateFormat ("yyyy-MM-dd");
Date start_date = ft.parse(startDateStr );
Date end_date = ft.parse(endDateStr);
// For each feature in the layer
JSONArray features = inputZoneFeatures.getJSONArray("features");
JSONArray feature_results = new JSONArray();
for (int i = 0; i < features.length(); i++) {
JSONObject feat = features.getJSONObject(i);
JSONObject props = feat.getJSONObject("properties");
String feature_id = props.has("id") ? "id" : props.has("gid") ? "gid" : "";
// Get the centroid of the feature.
JSONObject geom = feat.getJSONObject("geometry");
GISObject gisObject = GISObjectFactory.createGISObject(geom, null);
double[] centroid = gisObject.getCentroid();
int[] cell = null;
JSONArray climate_data = new JSONArray();
climate_data.put(getHeaderWithUnits());
for (String ncFileTemplate : ncFiles) {
// Read each variable's time slice as an array and store in a variable list.
ArrayList<Array> resultsByVariable = new ArrayList<Array>();
ArrayList<Variable> variables = new ArrayList<>();
CalendarDate nc_start_date = null;
for (String variable_name : parameters) {
if (variable_name.equals("date")) continue;
String ncFilePath = ncFileTemplate.replace("{param}", variable_name);
GridDataset ncGrid = GridDataset.open(ncFilePath);
NetcdfFile ncFile = ncGrid.getNetcdfFile();
if (ncGrid != null) {
List<GridDatatype> grids = ncGrid.getGrids();
GridCoordSystem gcs = grids.get(0).getCoordinateSystem();
cell = gcs.findXYindexFromLatLon(centroid[1], centroid[0], null);
FeatureDataset fd = (FeatureDataset)ncGrid;
nc_start_date = fd.getCalendarDateStart();
CalendarDate nc_end_date = fd.getCalendarDateEnd();
int start_date_index = getDateIndex(nc_start_date, start_date);
int end_date_index = getDateIndex(nc_start_date, end_date);
if (end_date_index > start_date_index) {
Variable v = findVariable(variable_name, ncFile);
if (v != null) {
LOG.info("Variable = " + v.toString());
int[] varShape = v.getShape();
if (varShape.length == 3) {
variables.add(v);
// Check range
if (end_date_index > v.getDimension(0).getLength()) {
end_date_index = v.getDimension(0).getLength()-1;
}
if (start_date_index < v.getDimension(0).getLength()) {
int[] origin = new int[] { start_date_index, cell[1], cell[0]};
int[] size = new int[]{end_date_index-start_date_index+1, 1, 1};
Array data1D = null;
data1D = v.read(origin, size);
resultsByVariable.add(data1D);
}
else {
// Request out of range.
int[] dim = new int[] { 0 };
Array data1D = new ArrayFloat(dim);
resultsByVariable.add(data1D);
}
}
}
}
}
}
// Transpose each variable's array values and append to a list of values by date.
Calendar current_date = Calendar.getInstance();
current_date.setTime(start_date);
Calendar nc_start_date_cal = Calendar.getInstance();
nc_start_date_cal.setTime(nc_start_date.toDate());
// Check if this netcdf file starts after the start date
if (nc_start_date_cal.after(current_date)) {
current_date.setTime(nc_start_date.toDate());
}
for (int itime=0; itime<resultsByVariable.get(0).getSize(); itime++) {
JSONArray row = new JSONArray();
row.put(ft.format(current_date.getTime()));
for (int ivar=1; ivar<parameters.size(); ivar++) {
String v = parameters.get(ivar);
Double val = resultsByVariable.get(ivar-1).getDouble(itime);
row.put(applyUnitsConversion(val, v));
}
climate_data.put(row);
current_date.add(Calendar.DATE, 1);
}
}
JSONObject feature_result = null;
feature_result = new JSONObject()
.put("netcdf_cell", Arrays.asList(cell[0], cell[1]))
.put(feature_id, props.get(feature_id))
.put("data", climate_data);
feature_results.put(feature_result);
}
return feature_results;
}
/**
* Calculate date offset. Assumes daily timestep.
* @param nc_start_date
* @param start_date
* @return
*/
public int getDateIndex(CalendarDate nc_start_date, Date start_date) {
DateRange dr = new DateRange(nc_start_date.toDate(), start_date);
TimeDuration duration = dr.getDuration();
double dur_seconds = duration.getValueInSeconds();
long start_date_index_l = Math.round(dur_seconds / 60 / 60 / 24);
int start_date_index = (int)start_date_index_l;
assert start_date_index == start_date_index_l;
return start_date_index;
}
public Variable findVariable(String variable_name, NetcdfFile ncFile) {
List<Variable> variables = ncFile.getVariables();
for (Variable var : variables) {
if (var.getFullName().contains(variable_name)) {
return var;
}
}
return null;
}
public JSONArray extractFromThreddsPython(String forecastModel, String forecastOption, JSONObject inputZoneFeatures,
String startDateStr, String endDateStr, String timestep) throws IOException, InterruptedException, JSONException, GISObjectException, ServiceException {
// For each feature in the layer
JSONArray features = inputZoneFeatures.getJSONArray("features");
JSONArray feature_results = new JSONArray();
for (int i = 0; i < features.length(); i++) {
JSONObject feat = features.getJSONObject(i);
JSONObject props = feat.getJSONObject("properties");
String feature_id = props.has("id") ? "id" : props.has("gid") ? "gid" : "";
// Get the centroid of the feature.
JSONObject geom = feat.getJSONObject("geometry");
GISObject gisObject = GISObjectFactory.createGISObject(geom, null);
double[] centroid = gisObject.getCentroid();
// Note that this isn't actually used yet in python because it is trickier
// to convert values.
List<String> varlist = new ArrayList<>();
for (String variable_name : parameters) {
if (!variable_name.equals("date")) {
// Get the conversion factor
Double cf = applyUnitsConversion(1.0, variable_name);
varlist.add(variable_name + ":" + "units" + ":" + cf.toString());
}
}
List<String> args = new ArrayList<>();
args.addAll(Arrays.asList(
"python", "/tmp/csip/bin/python/maca_extraction.py",
"--model", forecastModel, "--option", forecastOption,
"--lon", Double.toString(centroid[0]),
"--lat", Double.toString(centroid[1]),
"--start-date", startDateStr, "--end-date", endDateStr,
"--timestep", timestep));
args.addAll(varlist);
ProcessBuilder pb = new ProcessBuilder(args);
pb.redirectErrorStream(true);
pb.directory(getWorkspaceDir());
Process p = pb.start();
BufferedReader bfr = new BufferedReader(new InputStreamReader(p.getInputStream()));
int exitCode = p.waitFor();
System.out.println("Exit Code : " + exitCode);
String line;
List<String> lines = new ArrayList<>();
while ((line = bfr.readLine()) != null) {
System.out.println("Python Output: " + line);
lines.add(line);
}
if (exitCode > 0) {
throw new ServiceException("Failed to run python with args\n" +
String.join(" ", args) + "\n" +
"Output is \n" +
String.join("\n", lines));
}
// Add file to results
JSONArray climate_data = new JSONArray();
climate_data.put(getHeaderWithUnits());
File results = getWorkspaceFile("results.csv");
try (BufferedReader br = new BufferedReader(new FileReader(results))) {
while ((line = br.readLine()) != null) {
String[] tokens = line.split(",");
// Skip date
List<Object> converted_vals = new ArrayList<>();
converted_vals.add(tokens[0]);
for (int ivar=1; ivar<parameters.size(); ivar++) {
String variable_name = parameters.get(ivar);
// Get the conversion factor
Double val = Double.parseDouble(tokens[ivar]);
Double cf = applyUnitsConversion(val, variable_name);
converted_vals.add(cf);
}
climate_data.put(converted_vals);
}
}
File cellFile = getWorkspaceFile("cell.txt");
String cell = "";
try (BufferedReader br = new BufferedReader(new FileReader(cellFile))) {
while ((line = br.readLine()) != null) {
cell = line;
}
}
JSONObject feature_result = new JSONObject()
.put("netcdf_cell", cell)
.put(feature_id, props.get(feature_id))
.put("data", climate_data)
.put("geometry", feat);
feature_results.put(feature_result);
}
return feature_results;
}
}