pktools  2.6.5
Processing Kernel for geospatial data
pkstat.cc
1 /**********************************************************************
2 pkstat.cc: program to calculate basic statistics from raster dataset
3 Copyright (C) 2008-2015 Pieter Kempeneers
4 
5 This file is part of pktools
6 
7 pktools is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
11 
12 pktools is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16 
17 You should have received a copy of the GNU General Public License
18 along with pktools. If not, see <http://www.gnu.org/licenses/>.
19 ***********************************************************************/
20 #include <iostream>
21 #include <fstream>
22 #include <math.h>
23 #include "base/Optionpk.h"
24 #include "algorithms/StatFactory.h"
25 #include "algorithms/ImgRegression.h"
26 /******************************************************************************/
78 using namespace std;
79 
80 int main(int argc, char *argv[])
81 {
82  Optionpk<string> input_opt("i","input","name of the input raster dataset");
83  Optionpk<unsigned short> band_opt("b","band","band(s) on which to calculate statistics",0);
84  Optionpk<bool> filename_opt("f", "filename", "Shows image filename ", false);
85  Optionpk<bool> stat_opt("stats", "statistics", "Shows basic statistics (calculate in memory) (min,max, mean and stdDev of the raster datasets)", false);
86  Optionpk<bool> fstat_opt("fstats", "fstatistics", "Shows basic statistics using GDAL computeStatistics (min,max, mean and stdDev of the raster datasets)", false);
87  Optionpk<double> ulx_opt("ulx", "ulx", "Upper left x value bounding box");
88  Optionpk<double> uly_opt("uly", "uly", "Upper left y value bounding box");
89  Optionpk<double> lrx_opt("lrx", "lrx", "Lower right x value bounding box");
90  Optionpk<double> lry_opt("lry", "lry", "Lower right y value bounding box");
91  Optionpk<double> nodata_opt("nodata","nodata","Set nodata value(s)");
92  Optionpk<short> down_opt("down", "down", "Down sampling factor (for raster sample datasets only). Can be used to create grid points", 1);
93  Optionpk<unsigned int> random_opt("rnd", "rnd", "generate random numbers", 0);
94  Optionpk<double> scale_opt("scale", "scale", "Scale(s) for reading input image(s)");
95  Optionpk<double> offset_opt("offset", "offset", "Offset(s) for reading input image(s)");
96 
97  // Optionpk<bool> transpose_opt("t","transpose","transpose output",false);
98  // Optionpk<std::string> randdist_opt("dist", "dist", "distribution for generating random numbers, see http://www.gn/software/gsl/manual/gsl-ref_toc.html#TOC320 (only uniform and Gaussian supported yet)", "gaussian");
99  // Optionpk<double> randa_opt("rnda", "rnda", "first parameter for random distribution (mean value in case of Gaussian)", 0);
100  // Optionpk<double> randb_opt("rndb", "rndb", "second parameter for random distribution (standard deviation in case of Gaussian)", 1);
101  Optionpk<bool> mean_opt("mean","mean","calculate mean",false);
102  Optionpk<bool> median_opt("median","median","calculate median",false);
103  Optionpk<bool> var_opt("var","var","calculate variance",false);
104  Optionpk<bool> skewness_opt("skew","skewness","calculate skewness",false);
105  Optionpk<bool> kurtosis_opt("kurt","kurtosis","calculate kurtosis",false);
106  Optionpk<bool> stdev_opt("stdev","stdev","calculate standard deviation",false);
107  Optionpk<bool> sum_opt("sum","sum","calculate sum of column",false);
108  Optionpk<bool> minmax_opt("mm","minmax","calculate minimum and maximum value",false);
109  Optionpk<bool> min_opt("min","min","calculate minimum value",false);
110  Optionpk<bool> max_opt("max","max","calculate maximum value",false);
111  Optionpk<double> src_min_opt("src_min","src_min","start reading source from this minimum value");
112  Optionpk<double> src_max_opt("src_max","src_max","stop reading source from this maximum value");
113  Optionpk<bool> histogram_opt("hist","hist","calculate histogram",false);
114  Optionpk<bool> histogram2d_opt("hist2d","hist2d","calculate 2-dimensional histogram based on two images",false);
115  Optionpk<short> nbin_opt("nbin","nbin","number of bins to calculate histogram");
116  Optionpk<bool> relative_opt("rel","relative","use percentiles for histogram to calculate histogram",false);
117  Optionpk<bool> kde_opt("kde","kde","Use Kernel density estimation when producing histogram. The standard deviation is estimated based on Silverman's rule of thumb",false);
118  Optionpk<bool> rmse_opt("rmse","rmse","calculate root mean square error between two raster datasets",false);
119  Optionpk<bool> reg_opt("reg","regression","calculate linear regression between two raster datasets and get correlation coefficient",false);
120  Optionpk<bool> regerr_opt("regerr","regerr","calculate linear regression between two raster datasets and get root mean square error",false);
121  Optionpk<bool> preg_opt("preg","preg","calculate perpendicular regression between two raster datasets and get correlation coefficient",false);
122  Optionpk<short> verbose_opt("v", "verbose", "verbose mode when positive", 0,2);
123  fstat_opt.setHide(1);
124  ulx_opt.setHide(1);
125  uly_opt.setHide(1);
126  lrx_opt.setHide(1);
127  lry_opt.setHide(1);
128  down_opt.setHide(1);
129  random_opt.setHide(1);
130  scale_opt.setHide(1);
131  offset_opt.setHide(1);
132  src_min_opt.setHide(1);
133  src_max_opt.setHide(1);
134  kde_opt.setHide(1);
135 
136  // range_opt.setHide(1);
137  // transpose_opt.setHide(1);
138 
139  bool doProcess;//stop process when program was invoked with help option (-h --help)
140  try{
141  //mandatory options
142  doProcess=input_opt.retrieveOption(argc,argv);
143  //optional options
144  band_opt.retrieveOption(argc,argv);
145  filename_opt.retrieveOption(argc,argv);
146  stat_opt.retrieveOption(argc,argv);
147  fstat_opt.retrieveOption(argc,argv);
148  nodata_opt.retrieveOption(argc,argv);
149  mean_opt.retrieveOption(argc,argv);
150  median_opt.retrieveOption(argc,argv);
151  var_opt.retrieveOption(argc,argv);
152  stdev_opt.retrieveOption(argc,argv);
153  minmax_opt.retrieveOption(argc,argv);
154  min_opt.retrieveOption(argc,argv);
155  max_opt.retrieveOption(argc,argv);
156  histogram_opt.retrieveOption(argc,argv);
157  nbin_opt.retrieveOption(argc,argv);
158  relative_opt.retrieveOption(argc,argv);
159  histogram2d_opt.retrieveOption(argc,argv);
160  rmse_opt.retrieveOption(argc,argv);
161  reg_opt.retrieveOption(argc,argv);
162  regerr_opt.retrieveOption(argc,argv);
163  preg_opt.retrieveOption(argc,argv);
164  //advanced options
165  ulx_opt.retrieveOption(argc,argv);
166  uly_opt.retrieveOption(argc,argv);
167  lrx_opt.retrieveOption(argc,argv);
168  lry_opt.retrieveOption(argc,argv);
169  down_opt.retrieveOption(argc,argv);
170  random_opt.retrieveOption(argc,argv);
171  scale_opt.retrieveOption(argc,argv);
172  offset_opt.retrieveOption(argc,argv);
173  src_min_opt.retrieveOption(argc,argv);
174  src_max_opt.retrieveOption(argc,argv);
175  kde_opt.retrieveOption(argc,argv);
176  verbose_opt.retrieveOption(argc,argv);
177  }
178  catch(string predefinedString){
179  std::cout << predefinedString << std::endl;
180  exit(0);
181  }
182  if(!doProcess){
183  cout << endl;
184  cout << "Usage: pkstat -i input" << endl;
185  cout << endl;
186  std::cout << "short option -h shows basic options only, use long option --help to show all options" << std::endl;
187  exit(0);//help was invoked, stop processing
188  }
189 
190  if(src_min_opt.size()){
191  while(src_min_opt.size()<band_opt.size())
192  src_min_opt.push_back(src_min_opt[0]);
193  }
194  if(src_max_opt.size()){
195  while(src_max_opt.size()<band_opt.size())
196  src_max_opt.push_back(src_max_opt[0]);
197  }
198 
199  unsigned int nbin=0;
200  double minX=0;
201  double minY=0;
202  double maxX=0;
203  double maxY=0;
204  double minValue=0;
205  double maxValue=0;
206  double meanValue=0;
207  double medianValue=0;
208  double stdDev=0;
209 
210  const char* pszMessage;
211  void* pProgressArg=NULL;
212  GDALProgressFunc pfnProgress=GDALTermProgress;
213  double progress=0;
214  srand(time(NULL));
215 
218  std::vector<double> histogramOutput;
219  double nsample=0;
220 
221  ImgReaderGdal imgReader;
222 
223  if(scale_opt.size()){
224  while(scale_opt.size()<input_opt.size())
225  scale_opt.push_back(scale_opt[0]);
226  }
227  if(offset_opt.size()){
228  while(offset_opt.size()<input_opt.size())
229  offset_opt.push_back(offset_opt[0]);
230  }
231  if(input_opt.empty()){
232  std::cerr << "No image dataset provided (use option -i). Use --help for help information";
233  exit(0);
234  }
235  for(int ifile=0;ifile<input_opt.size();++ifile){
236  try{
237  imgReader.open(input_opt[ifile]);
238  }
239  catch(std::string errorstring){
240  std::cout << errorstring << std::endl;
241  exit(0);
242  }
243 
244  if(filename_opt[0])
245  std::cout << " --input " << input_opt[ifile] << " ";
246 
247  for(int inodata=0;inodata<nodata_opt.size();++inodata)
248  imgReader.pushNoDataValue(nodata_opt[inodata]);
249 
250  int nband=band_opt.size();
251  for(int iband=0;iband<nband;++iband){
252 
253  for(int inodata=0;inodata<nodata_opt.size();++inodata){
254  if(!inodata)
255  imgReader.GDALSetNoDataValue(nodata_opt[0],band_opt[iband]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
256  }
257 
258  if(offset_opt.size()>ifile)
259  imgReader.setOffset(offset_opt[ifile],band_opt[iband]);
260  if(scale_opt.size()>ifile)
261  imgReader.setScale(scale_opt[ifile],band_opt[iband]);
262 
263  if(stat_opt[0]||mean_opt[0]||median_opt[0]||var_opt[0]||stdev_opt[0]){//the hard way (in memory)
264  //todo: take src_min and src_max into account!
266  vector<double> readBuffer;
267  double varValue;
268  imgReader.readDataBlock(readBuffer, GDT_Float64, 0, imgReader.nrOfCol()-1, 0, imgReader.nrOfRow()-1, band_opt[0]);
269  stat.setNoDataValues(nodata_opt);
270  stat.meanVar(readBuffer,meanValue,varValue);
271  medianValue=stat.median(readBuffer);
272  stat.minmax(readBuffer,readBuffer.begin(),readBuffer.end(),minValue,maxValue);
273  if(mean_opt[0])
274  std::cout << "--mean " << meanValue << " ";
275  if(median_opt[0])
276  std::cout << "--median " << medianValue << " ";
277  if(stdev_opt[0])
278  std::cout << "--stdDev " << sqrt(varValue) << " ";
279  if(var_opt[0])
280  std::cout << "--var " << varValue << " ";
281  if(stat_opt[0])
282  std::cout << "-min " << minValue << " -max " << maxValue << " --mean " << meanValue << " --stdDev " << sqrt(varValue) << " ";
283  }
284 
285  if(fstat_opt[0]){//the fast way
286  assert(band_opt[iband]<imgReader.nrOfBand());
287  GDALProgressFunc pfnProgress;
288  void* pProgressData;
289  GDALRasterBand* rasterBand;
290  rasterBand=imgReader.getRasterBand(band_opt[iband]);
291  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev,pfnProgress,pProgressData);
292 
293  std::cout << "-min " << minValue << " -max " << maxValue << " --mean " << meanValue << " --stdDev " << stdDev << " ";
294  }
295 
296  if(minmax_opt[0]||min_opt[0]||max_opt[0]){
297  assert(band_opt[iband]<imgReader.nrOfBand());
298 
299  if((ulx_opt.size()||uly_opt.size()||lrx_opt.size()||lry_opt.size())&&(imgReader.covers(ulx_opt[0],uly_opt[0],lrx_opt[0],lry_opt[0]))){
300  double uli,ulj,lri,lrj;
301  imgReader.geo2image(ulx_opt[0],uly_opt[0],uli,ulj);
302  imgReader.geo2image(lrx_opt[0],lry_opt[0],lri,lrj);
303  imgReader.getMinMax(static_cast<int>(uli),static_cast<int>(lri),static_cast<int>(ulj),static_cast<int>(lrj),band_opt[iband],minValue,maxValue);
304  }
305  else{
306  imgReader.getMinMax(minValue,maxValue,band_opt[iband],true);
307  }
308  if(minmax_opt[0])
309  std::cout << "-min " << minValue << " -max " << maxValue << " ";
310  else{
311  if(min_opt[0])
312  std::cout << "-min " << minValue << " ";
313  if(max_opt[0])
314  std::cout << "-max " << maxValue << " ";
315  }
316  }
317  }
318  if(histogram_opt[0]){//aggregate results from multiple inputs, but only calculate for first selected band
319  assert(band_opt[0]<imgReader.nrOfBand());
320  nbin=(nbin_opt.size())? nbin_opt[0]:0;
321 
322  imgReader.getMinMax(minValue,maxValue,band_opt[0]);
323  if(src_min_opt.size())
324  minValue=src_min_opt[0];
325  if(src_max_opt.size())
326  maxValue=src_max_opt[0];
327  if(minValue>=maxValue)
328  imgReader.getMinMax(minValue,maxValue,band_opt[0]);
329 
330  if(verbose_opt[0])
331  cout << "number of valid pixels in image: " << imgReader.getNvalid(band_opt[0]) << endl;
332 
333  nsample+=imgReader.getHistogram(histogramOutput,minValue,maxValue,nbin,band_opt[0],kde_opt[0]);
334 
335  //only output for last input file
336  if(ifile==input_opt.size()-1){
337  std::cout.precision(10);
338  for(int bin=0;bin<nbin;++bin){
339  double binValue=0;
340  if(nbin==maxValue-minValue+1)
341  binValue=minValue+bin;
342  else
343  binValue=minValue+static_cast<double>(maxValue-minValue)*(bin+0.5)/nbin;
344  std::cout << binValue << " ";
345  if(relative_opt[0]||kde_opt[0])
346  std::cout << 100.0*static_cast<double>(histogramOutput[bin])/static_cast<double>(nsample) << std::endl;
347  else
348  std::cout << static_cast<double>(histogramOutput[bin]) << std::endl;
349  }
350  }
351  }
352  if(histogram2d_opt[0]&&input_opt.size()<2){
353  assert(band_opt.size()>1);
354  imgReader.getMinMax(minX,maxX,band_opt[0]);
355  imgReader.getMinMax(minY,maxY,band_opt[1]);
356  if(src_min_opt.size()){
357  minX=src_min_opt[0];
358  minY=src_min_opt[1];
359  }
360  if(src_max_opt.size()){
361  maxX=src_max_opt[0];
362  maxY=src_max_opt[1];
363  }
364  nbin=(nbin_opt.size())? nbin_opt[0]:0;
365  if(nbin<=1){
366  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
367  if(minX>=maxX)
368  imgReader.getMinMax(minX,maxX,band_opt[0]);
369  if(minY>=maxY)
370  imgReader.getMinMax(minY,maxY,band_opt[1]);
371 
372  minValue=(minX<minY)? minX:minY;
373  maxValue=(maxX>maxY)? maxX:maxY;
374  if(verbose_opt[0])
375  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
376  nbin=maxValue-minValue+1;
377  }
378  assert(nbin>1);
379  double sigma=0;
380  //kernel density estimation as in http://en.wikipedia.org/wiki/Kernel_density_estimation
381  if(kde_opt[0]){
382  assert(band_opt[0]<imgReader.nrOfBand());
383  assert(band_opt[1]<imgReader.nrOfBand());
384  GDALProgressFunc pfnProgress;
385  void* pProgressData;
386  GDALRasterBand* rasterBand;
387  double stdDev1=0;
388  double stdDev2=0;
389  rasterBand=imgReader.getRasterBand(band_opt[0]);
390  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev1,pfnProgress,pProgressData);
391  rasterBand=imgReader.getRasterBand(band_opt[1]);
392  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev2,pfnProgress,pProgressData);
393 
394  double estimatedSize=1.0*imgReader.getNvalid(band_opt[0])/down_opt[0]/down_opt[0];
395  if(random_opt[0]>0)
396  estimatedSize*=random_opt[0]/100.0;
397  sigma=1.06*sqrt(stdDev1*stdDev2)*pow(estimatedSize,-0.2);
398  }
399  assert(nbin);
400  if(verbose_opt[0]){
401  if(sigma>0)
402  std::cout << "calculating 2d kernel density estimate with sigma " << sigma << " for bands " << band_opt[0] << " and " << band_opt[1] << std::endl;
403  else
404  std::cout << "calculating 2d histogram for bands " << band_opt[0] << " and " << band_opt[1] << std::endl;
405  std::cout << "nbin: " << nbin << std::endl;
406  }
407 
408 
409  vector< vector<double> > output;
410 
411  if(maxX<=minX)
412  imgReader.getMinMax(minX,maxX,band_opt[0]);
413  if(maxY<=minY)
414  imgReader.getMinMax(minY,maxY,band_opt[1]);
415 
416  if(maxX<=minX){
417  std::ostringstream s;
418  s<<"Error: could not calculate distribution (minX>=maxX)";
419  throw(s.str());
420  }
421  if(maxY<=minY){
422  std::ostringstream s;
423  s<<"Error: could not calculate distribution (minY>=maxY)";
424  throw(s.str());
425  }
426  output.resize(nbin);
427  for(int i=0;i<nbin;++i){
428  output[i].resize(nbin);
429  for(int j=0;j<nbin;++j)
430  output[i][j]=0;
431  }
432  int binX=0;
433  int binY=0;
434  vector<double> inputX(imgReader.nrOfCol());
435  vector<double> inputY(imgReader.nrOfCol());
436  unsigned long int nvalid=0;
437  for(int irow=0;irow<imgReader.nrOfRow();++irow){
438  if(irow%down_opt[0])
439  continue;
440  imgReader.readData(inputX,GDT_Float64,irow,band_opt[0]);
441  imgReader.readData(inputY,GDT_Float64,irow,band_opt[1]);
442  for(int icol=0;icol<imgReader.nrOfCol();++icol){
443  if(icol%down_opt[0])
444  continue;
445  if(random_opt[0]>0){
446  double p=static_cast<double>(rand())/(RAND_MAX);
447  p*=100.0;
448  if(p>random_opt[0])
449  continue;//do not select for now, go to next column
450  }
451  if(imgReader.isNoData(inputX[icol]))
452  continue;
453  if(imgReader.isNoData(inputY[icol]))
454  continue;
455  ++nvalid;
456  if(inputX[icol]>=maxX)
457  binX=nbin-1;
458  else if(inputX[icol]<=minX)
459  binX=0;
460  else
461  binX=static_cast<int>(static_cast<double>(inputX[icol]-minX)/(maxX-minX)*nbin);
462  if(inputY[icol]>=maxY)
463  binY=nbin-1;
464  else if(inputY[icol]<=minX)
465  binY=0;
466  else
467  binY=static_cast<int>(static_cast<double>(inputY[icol]-minY)/(maxY-minY)*nbin);
468  assert(binX>=0);
469  assert(binX<output.size());
470  assert(binY>=0);
471  assert(binY<output[binX].size());
472  if(sigma>0){
473  //create kde for Gaussian basis function
474  //todo: speed up by calculating first and last bin with non-zero contriubtion...
475  for(int ibinX=0;ibinX<nbin;++ibinX){
476  double centerX=minX+static_cast<double>(maxX-minX)*ibinX/nbin;
477  double pdfX=gsl_ran_gaussian_pdf(inputX[icol]-centerX, sigma);
478  for(int ibinY=0;ibinY<nbin;++ibinY){
479  //calculate \integral_ibinX^(ibinX+1)
480  double centerY=minY+static_cast<double>(maxY-minY)*ibinY/nbin;
481  double pdfY=gsl_ran_gaussian_pdf(inputY[icol]-centerY, sigma);
482  output[ibinX][binY]+=pdfX*pdfY;
483  }
484  }
485  }
486  else
487  ++output[binX][binY];
488  }
489  }
490  if(verbose_opt[0])
491  cout << "number of valid pixels: " << nvalid << endl;
492 
493  for(int binX=0;binX<nbin;++binX){
494  cout << endl;
495  for(int binY=0;binY<nbin;++binY){
496  double binValueX=0;
497  if(nbin==maxX-minX+1)
498  binValueX=minX+binX;
499  else
500  binValueX=minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
501  double binValueY=0;
502  if(nbin==maxY-minY+1)
503  binValueY=minY+binY;
504  else
505  binValueY=minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
506 
507  double value=static_cast<double>(output[binX][binY]);
508 
509  if(relative_opt[0])
510  value*=100.0/nvalid;
511 
512  cout << binValueX << " " << binValueY << " " << value << std::endl;
513  // double value=static_cast<double>(output[binX][binY])/nvalid;
514  // cout << (maxX-minX)*bin/(nbin-1)+minX << " " << (maxY-minY)*bin/(nbin-1)+minY << " " << value << std::endl;
515  }
516  }
517  }
518  if(reg_opt[0]&&input_opt.size()<2){
519  if(band_opt.size()<2)
520  continue;
521  imgreg.setDown(down_opt[0]);
522  imgreg.setThreshold(random_opt[0]);
523  double c0=0;//offset
524  double c1=1;//scale
525  double r2=imgreg.getR2(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
526  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
527  }
528  if(regerr_opt[0]&&input_opt.size()<2){
529  if(band_opt.size()<2)
530  continue;
531  imgreg.setDown(down_opt[0]);
532  imgreg.setThreshold(random_opt[0]);
533  double c0=0;//offset
534  double c1=1;//scale
535  double err=imgreg.getRMSE(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
536  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -rmse " << err << std::endl;
537  }
538  if(rmse_opt[0]&&input_opt.size()<2){
539  if(band_opt.size()<2)
540  continue;
541  imgreg.setDown(down_opt[0]);
542  imgreg.setThreshold(random_opt[0]);
543  double c0=0;//offset
544  double c1=1;//scale
545  double err=imgreg.getRMSE(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
546  std::cout << " -rmse " << err << std::endl;
547  }
548  if(preg_opt[0]&&input_opt.size()<2){
549  if(band_opt.size()<2)
550  continue;
551  imgreg.setDown(down_opt[0]);
552  imgreg.setThreshold(random_opt[0]);
553  double c0=0;//offset
554  double c1=1;//scale
555  double r2=imgreg.pgetR2(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
556  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
557  }
558  imgReader.close();
559  }
560  if(reg_opt[0]&&(input_opt.size()>1)){
561  imgreg.setDown(down_opt[0]);
562  imgreg.setThreshold(random_opt[0]);
563  double c0=0;//offset
564  double c1=1;//scale
565  while(band_opt.size()<input_opt.size())
566  band_opt.push_back(band_opt[0]);
567  if(src_min_opt.size()){
568  while(src_min_opt.size()<input_opt.size())
569  src_min_opt.push_back(src_min_opt[0]);
570  }
571  if(src_max_opt.size()){
572  while(src_max_opt.size()<input_opt.size())
573  src_max_opt.push_back(src_max_opt[0]);
574  }
575  ImgReaderGdal imgReader1(input_opt[0]);
576  ImgReaderGdal imgReader2(input_opt[1]);
577 
578  if(offset_opt.size())
579  imgReader1.setOffset(offset_opt[0],band_opt[0]);
580  if(scale_opt.size())
581  imgReader1.setScale(scale_opt[0],band_opt[0]);
582  if(offset_opt.size()>1)
583  imgReader2.setOffset(offset_opt[1],band_opt[1]);
584  if(scale_opt.size()>1)
585  imgReader2.setScale(scale_opt[1],band_opt[1]);
586 
587  for(int inodata=0;inodata<nodata_opt.size();++inodata){
588  if(!inodata){
589  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
590  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
591  }
592  imgReader1.pushNoDataValue(nodata_opt[inodata]);
593  imgReader2.pushNoDataValue(nodata_opt[inodata]);
594  }
595 
596  double r2=imgreg.getR2(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
597  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
598  imgReader1.close();
599  imgReader2.close();
600  }
601  if(preg_opt[0]&&(input_opt.size()>1)){
602  imgreg.setDown(down_opt[0]);
603  imgreg.setThreshold(random_opt[0]);
604  double c0=0;//offset
605  double c1=1;//scale
606  while(band_opt.size()<input_opt.size())
607  band_opt.push_back(band_opt[0]);
608  if(src_min_opt.size()){
609  while(src_min_opt.size()<input_opt.size())
610  src_min_opt.push_back(src_min_opt[0]);
611  }
612  if(src_max_opt.size()){
613  while(src_max_opt.size()<input_opt.size())
614  src_max_opt.push_back(src_max_opt[0]);
615  }
616  ImgReaderGdal imgReader1(input_opt[0]);
617  ImgReaderGdal imgReader2(input_opt[1]);
618 
619  if(offset_opt.size())
620  imgReader1.setOffset(offset_opt[0],band_opt[0]);
621  if(scale_opt.size())
622  imgReader1.setScale(scale_opt[0],band_opt[0]);
623  if(offset_opt.size()>1)
624  imgReader2.setOffset(offset_opt[1],band_opt[1]);
625  if(scale_opt.size()>1)
626  imgReader2.setScale(scale_opt[1],band_opt[1]);
627 
628  for(int inodata=0;inodata<nodata_opt.size();++inodata){
629  if(!inodata){
630  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
631  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
632  }
633  imgReader1.pushNoDataValue(nodata_opt[inodata]);
634  imgReader2.pushNoDataValue(nodata_opt[inodata]);
635  }
636 
637  double r2=imgreg.pgetR2(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
638  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
639  imgReader1.close();
640  imgReader2.close();
641  }
642  if(regerr_opt[0]&&(input_opt.size()>1)){
643  imgreg.setDown(down_opt[0]);
644  imgreg.setThreshold(random_opt[0]);
645  double c0=0;//offset
646  double c1=1;//scale
647  while(band_opt.size()<input_opt.size())
648  band_opt.push_back(band_opt[0]);
649  if(src_min_opt.size()){
650  while(src_min_opt.size()<input_opt.size())
651  src_min_opt.push_back(src_min_opt[0]);
652  }
653  if(src_max_opt.size()){
654  while(src_max_opt.size()<input_opt.size())
655  src_max_opt.push_back(src_max_opt[0]);
656  }
657  ImgReaderGdal imgReader1(input_opt[0]);
658  ImgReaderGdal imgReader2(input_opt[1]);
659 
660  if(offset_opt.size())
661  imgReader1.setOffset(offset_opt[0],band_opt[0]);
662  if(scale_opt.size())
663  imgReader1.setScale(scale_opt[0],band_opt[0]);
664  if(offset_opt.size()>1)
665  imgReader2.setOffset(offset_opt[1],band_opt[1]);
666  if(scale_opt.size()>1)
667  imgReader2.setScale(scale_opt[1],band_opt[1]);
668 
669  for(int inodata=0;inodata<nodata_opt.size();++inodata){
670  if(!inodata){
671  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
672  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
673  }
674  imgReader1.pushNoDataValue(nodata_opt[inodata]);
675  imgReader2.pushNoDataValue(nodata_opt[inodata]);
676  }
677 
678  double err=imgreg.getRMSE(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
679  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -rmse " << err << std::endl;
680  imgReader1.close();
681  imgReader2.close();
682  }
683  if(rmse_opt[0]&&(input_opt.size()>1)){
684  imgreg.setDown(down_opt[0]);
685  imgreg.setThreshold(random_opt[0]);
686  double c0=0;//offset
687  double c1=1;//scale
688  while(band_opt.size()<input_opt.size())
689  band_opt.push_back(band_opt[0]);
690  if(src_min_opt.size()){
691  while(src_min_opt.size()<input_opt.size())
692  src_min_opt.push_back(src_min_opt[0]);
693  }
694  if(src_max_opt.size()){
695  while(src_max_opt.size()<input_opt.size())
696  src_max_opt.push_back(src_max_opt[0]);
697  }
698  ImgReaderGdal imgReader1(input_opt[0]);
699  ImgReaderGdal imgReader2(input_opt[1]);
700 
701  if(offset_opt.size())
702  imgReader1.setOffset(offset_opt[0],band_opt[0]);
703  if(scale_opt.size())
704  imgReader1.setScale(scale_opt[0],band_opt[0]);
705  if(offset_opt.size()>1)
706  imgReader2.setOffset(offset_opt[1],band_opt[1]);
707  if(scale_opt.size()>1)
708  imgReader2.setScale(scale_opt[1],band_opt[1]);
709 
710  for(int inodata=0;inodata<nodata_opt.size();++inodata){
711  if(!inodata){
712  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
713  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
714  }
715  imgReader1.pushNoDataValue(nodata_opt[inodata]);
716  imgReader2.pushNoDataValue(nodata_opt[inodata]);
717  }
718 
719  double err=imgreg.getRMSE(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
720  std::cout << "-rmse " << err << std::endl;
721  imgReader1.close();
722  imgReader2.close();
723  }
724  if(histogram2d_opt[0]&&(input_opt.size()>1)){
725  while(band_opt.size()<input_opt.size())
726  band_opt.push_back(band_opt[0]);
727  if(src_min_opt.size()){
728  while(src_min_opt.size()<input_opt.size())
729  src_min_opt.push_back(src_min_opt[0]);
730  }
731  if(src_max_opt.size()){
732  while(src_max_opt.size()<input_opt.size())
733  src_max_opt.push_back(src_max_opt[0]);
734  }
735  ImgReaderGdal imgReader1(input_opt[0]);
736  ImgReaderGdal imgReader2(input_opt[1]);
737 
738  if(offset_opt.size())
739  imgReader1.setOffset(offset_opt[0],band_opt[0]);
740  if(scale_opt.size())
741  imgReader1.setScale(scale_opt[0],band_opt[0]);
742  if(offset_opt.size()>1)
743  imgReader2.setOffset(offset_opt[1],band_opt[1]);
744  if(scale_opt.size()>1)
745  imgReader2.setScale(scale_opt[1],band_opt[1]);
746 
747  for(int inodata=0;inodata<nodata_opt.size();++inodata){
748  if(!inodata){
749  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
750  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
751  }
752  imgReader1.pushNoDataValue(nodata_opt[inodata]);
753  imgReader2.pushNoDataValue(nodata_opt[inodata]);
754  }
755 
756  imgReader1.getMinMax(minX,maxX,band_opt[0]);
757  imgReader2.getMinMax(minY,maxY,band_opt[1]);
758 
759  if(verbose_opt[0]){
760  cout << "minX: " << minX << endl;
761  cout << "maxX: " << maxX << endl;
762  cout << "minY: " << minY << endl;
763  cout << "maxY: " << maxY << endl;
764  }
765 
766  if(src_min_opt.size()){
767  minX=src_min_opt[0];
768  minY=src_min_opt[1];
769  }
770  if(src_max_opt.size()){
771  maxX=src_max_opt[0];
772  maxY=src_max_opt[1];
773  }
774 
775  nbin=(nbin_opt.size())? nbin_opt[0]:0;
776  if(nbin<=1){
777  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
778  // imgReader1.getMinMax(minX,maxX,band_opt[0]);
779  // imgReader2.getMinMax(minY,maxY,band_opt[0]);
780  if(minX>=maxX)
781  imgReader1.getMinMax(minX,maxX,band_opt[0]);
782  if(minY>=maxY)
783  imgReader2.getMinMax(minY,maxY,band_opt[1]);
784 
785  minValue=(minX<minY)? minX:minY;
786  maxValue=(maxX>maxY)? maxX:maxY;
787  if(verbose_opt[0])
788  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
789  nbin=maxValue-minValue+1;
790  }
791  assert(nbin>1);
792  double sigma=0;
793  //kernel density estimation as in http://en.wikipedia.org/wiki/Kernel_density_estimation
794  if(kde_opt[0]){
795  GDALProgressFunc pfnProgress;
796  void* pProgressData;
797  GDALRasterBand* rasterBand;
798  double stdDev1=0;
799  double stdDev2=0;
800  rasterBand=imgReader1.getRasterBand(band_opt[0]);
801  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev1,pfnProgress,pProgressData);
802  rasterBand=imgReader2.getRasterBand(band_opt[0]);
803  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev2,pfnProgress,pProgressData);
804 
805  //todo: think of smarter way how to estimate size (nodata!)
806  double estimatedSize=1.0*imgReader.getNvalid(band_opt[0])/down_opt[0]/down_opt[0];
807  if(random_opt[0]>0)
808  estimatedSize*=random_opt[0]/100.0;
809  sigma=1.06*sqrt(stdDev1*stdDev2)*pow(estimatedSize,-0.2);
810  }
811  assert(nbin);
812  if(verbose_opt[0]){
813  if(sigma>0)
814  std::cout << "calculating 2d kernel density estimate with sigma " << sigma << " for datasets " << input_opt[0] << " and " << input_opt[1] << std::endl;
815  else
816  std::cout << "calculating 2d histogram for datasets " << input_opt[0] << " and " << input_opt[1] << std::endl;
817  std::cout << "nbin: " << nbin << std::endl;
818  }
819 
820  vector< vector<double> > output;
821 
822  if(maxX<=minX)
823  imgReader1.getMinMax(minX,maxX,band_opt[0]);
824  if(maxY<=minY)
825  imgReader2.getMinMax(minY,maxY,band_opt[1]);
826 
827  if(maxX<=minX){
828  std::ostringstream s;
829  s<<"Error: could not calculate distribution (minX>=maxX)";
830  throw(s.str());
831  }
832  if(maxY<=minY){
833  std::ostringstream s;
834  s<<"Error: could not calculate distribution (minY>=maxY)";
835  throw(s.str());
836  }
837  if(verbose_opt[0]){
838  cout << "minX: " << minX << endl;
839  cout << "maxX: " << maxX << endl;
840  cout << "minY: " << minY << endl;
841  cout << "maxY: " << maxY << endl;
842  }
843  output.resize(nbin);
844  for(int i=0;i<nbin;++i){
845  output[i].resize(nbin);
846  for(int j=0;j<nbin;++j)
847  output[i][j]=0;
848  }
849  int binX=0;
850  int binY=0;
851  vector<double> inputX(imgReader1.nrOfCol());
852  vector<double> inputY(imgReader2.nrOfCol());
853  double nvalid=0;
854  double geoX=0;
855  double geoY=0;
856  double icol1=0;
857  double irow1=0;
858  double icol2=0;
859  double irow2=0;
860  for(int irow=0;irow<imgReader1.nrOfRow();++irow){
861  if(irow%down_opt[0])
862  continue;
863  irow1=irow;
864  imgReader1.image2geo(icol1,irow1,geoX,geoY);
865  imgReader2.geo2image(geoX,geoY,icol2,irow2);
866  irow2=static_cast<int>(irow2);
867  imgReader1.readData(inputX,GDT_Float64,irow1,band_opt[0]);
868  imgReader2.readData(inputY,GDT_Float64,irow2,band_opt[1]);
869  for(int icol=0;icol<imgReader.nrOfCol();++icol){
870  if(icol%down_opt[0])
871  continue;
872  icol1=icol;
873  if(random_opt[0]>0){
874  double p=static_cast<double>(rand())/(RAND_MAX);
875  p*=100.0;
876  if(p>random_opt[0])
877  continue;//do not select for now, go to next column
878  }
879  if(imgReader1.isNoData(inputX[icol]))
880  continue;
881  imgReader1.image2geo(icol1,irow1,geoX,geoY);
882  imgReader2.geo2image(geoX,geoY,icol2,irow2);
883  icol2=static_cast<int>(icol2);
884  if(imgReader2.isNoData(inputY[icol2]))
885  continue;
886  // ++nvalid;
887  if(inputX[icol1]>=maxX)
888  binX=nbin-1;
889  else if(inputX[icol]<=minX)
890  binX=0;
891  else
892  binX=static_cast<int>(static_cast<double>(inputX[icol1]-minX)/(maxX-minX)*nbin);
893  if(inputY[icol2]>=maxY)
894  binY=nbin-1;
895  else if(inputY[icol2]<=minY)
896  binY=0;
897  else
898  binY=static_cast<int>(static_cast<double>(inputY[icol2]-minY)/(maxY-minY)*nbin);
899  assert(binX>=0);
900  assert(binX<output.size());
901  assert(binY>=0);
902  assert(binY<output[binX].size());
903  if(sigma>0){
904  //create kde for Gaussian basis function
905  //todo: speed up by calculating first and last bin with non-zero contriubtion...
906  for(int ibinX=0;ibinX<nbin;++ibinX){
907  double centerX=minX+static_cast<double>(maxX-minX)*ibinX/nbin;
908  double pdfX=gsl_ran_gaussian_pdf(inputX[icol1]-centerX, sigma);
909  for(int ibinY=0;ibinY<nbin;++ibinY){
910  //calculate \integral_ibinX^(ibinX+1)
911  double centerY=minY+static_cast<double>(maxY-minY)*ibinY/nbin;
912  double pdfY=gsl_ran_gaussian_pdf(inputY[icol2]-centerY, sigma);
913  output[ibinX][binY]+=pdfX*pdfY;
914  nvalid+=pdfX*pdfY;
915  }
916  }
917  }
918  else{
919  ++output[binX][binY];
920  ++nvalid;
921  }
922  }
923  }
924  if(verbose_opt[0])
925  cout << "number of valid pixels: " << nvalid << endl;
926  for(int binX=0;binX<nbin;++binX){
927  cout << endl;
928  for(int binY=0;binY<nbin;++binY){
929  double binValueX=0;
930  if(nbin==maxX-minX+1)
931  binValueX=minX+binX;
932  else
933  binValueX=minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
934  double binValueY=0;
935  if(nbin==maxY-minY+1)
936  binValueY=minY+binY;
937  else
938  binValueY=minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
939  double value=static_cast<double>(output[binX][binY]);
940 
941  if(relative_opt[0]||kde_opt[0])
942  value*=100.0/nvalid;
943 
944  cout << binValueX << " " << binValueY << " " << value << std::endl;
945  // double value=static_cast<double>(output[binX][binY])/nvalid;
946  // cout << (maxX-minX)*bin/(nbin-1)+minX << " " << (maxY-minY)*bin/(nbin-1)+minY << " " << value << std::endl;
947  }
948  }
949  imgReader1.close();
950  imgReader2.close();
951  }
952 
953  if(!histogram_opt[0]||histogram2d_opt[0])
954  std::cout << std::endl;
955 }
956 
957 // int nband=(band_opt.size()) ? band_opt.size() : imgReader.nrOfBand();
958 
959 // const char* pszMessage;
960 // void* pProgressArg=NULL;
961 // GDALProgressFunc pfnProgress=GDALTermProgress;
962 // double progress=0;
963 // srand(time(NULL));
964 
965 
966 // statfactory::StatFactory stat;
967 // imgregression::ImgRegression imgreg;
968 
969 // pfnProgress(progress,pszMessage,pProgressArg);
970 // for(irow=0;irow<classReader.nrOfRow();++irow){
971 // if(irow%down_opt[0])
972 // continue;
973 // // classReader.readData(classBuffer,GDT_Int32,irow);
974 // classReader.readData(classBuffer,GDT_Float64,irow);
975 // double x,y;//geo coordinates
976 // double iimg,jimg;//image coordinates in img image
977 // for(icol=0;icol<classReader.nrOfCol();++icol){
978 // if(icol%down_opt[0])
979  // continue;
980 
981 
982  // if(rand_opt[0]>0){
983  // gsl_rng* r=stat.getRandomGenerator(time(NULL));
984  // //todo: init random number generator using time...
985  // if(verbose_opt[0])
986  // std::cout << "generating " << rand_opt[0] << " random numbers: " << std::endl;
987  // for(unsigned int i=0;i<rand_opt[0];++i)
988  // std::cout << i << " " << stat.getRandomValue(r,randdist_opt[0],randa_opt[0],randb_opt[0]) << std::endl;
989  // }
990 
991  // imgreg.setDown(down_opt[0]);
992  // imgreg.setThreshold(threshold_opt[0]);
993  // double c0=0;//offset
994  // double c1=1;//scale
995  // double err=uncertNodata_opt[0];//start with high initial value in case we do not have first ob err=imgreg.getRMSE(imgReaderModel1,imgReader,c0,c1,verbose_opt[0]);
996 
997  // int nband=band_opt.size();
998  // if(band_opt[0]<0)
999  // nband=imgReader.nrOfBand();
1000  // for(int iband=0;iband<nband;++iband){
1001  // unsigned short band_opt[iband]=(band_opt[0]<0)? iband : band_opt[iband];
1002 
1003  // if(minmax_opt[0]||min_opt[0]||max_opt[0]){
1004  // assert(band_opt[iband]<imgReader.nrOfBand());
1005  // if((ulx_opt.size()||uly_opt.size()||lrx_opt.size()||lry_opt.size())&&(imgReader.covers(ulx_opt[0],uly_opt[0],lrx_opt[0],lry_opt[0]))){
1006  // double uli,ulj,lri,lrj;
1007  // imgReader.geo2image(ulx_opt[0],uly_opt[0],uli,ulj);
1008  // imgReader.geo2image(lrx_opt[0],lry_opt[0],lri,lrj);
1009  // imgReader.getMinMax(static_cast<int>(uli),static_cast<int>(lri),static_cast<int>(ulj),static_cast<int>(lrj),band_opt[iband],minValue,maxValue);
1010  // }
1011  // else
1012  // imgReader.getMinMax(minValue,maxValue,band_opt[iband],true);
1013  // if(minmax_opt[0])
1014  // std::cout << "-min " << minValue << " -max " << maxValue << " ";
1015  // else{
1016  // if(min_opt[0])
1017  // std::cout << "-min " << minValue << " ";
1018  // if(max_opt[0])
1019  // std::cout << "-max " << maxValue << " ";
1020  // }
1021  // }
1022  // }
1023  // if(relative_opt[0])
1024  // hist_opt[0]=true;
1025  // if(hist_opt[0]){
1026  // assert(band_opt[0]<imgReader.nrOfBand());
1027  // unsigned int nbin=(nbin_opt.size())? nbin_opt[0]:0;
1028  // std::vector<unsigned long int> output;
1029  // minValue=0;
1030  // maxValue=0;
1031  // //todo: optimize such that getMinMax is only called once...
1032  // imgReader.getMinMax(minValue,maxValue,band_opt[0]);
1033 
1034  // if(src_min_opt.size())
1035  // minValue=src_min_opt[0];
1036  // if(src_max_opt.size())
1037  // maxValue=src_max_opt[0];
1038  // unsigned long int nsample=imgReader.getHistogram(output,minValue,maxValue,nbin,band_opt[0]);
1039  // std::cout.precision(10);
1040  // for(int bin=0;bin<nbin;++bin){
1041  // double binValue=0;
1042  // if(nbin==maxValue-minValue+1)
1043  // binValue=minValue+bin;
1044  // else
1045  // binValue=minValue+static_cast<double>(maxValue-minValue)*(bin+0.5)/nbin;
1046  // std::cout << binValue << " ";
1047  // if(relative_opt[0])
1048  // std::cout << 100.0*static_cast<double>(output[bin])/static_cast<double>(nsample) << std::endl;
1049  // else
1050  // std::cout << static_cast<double>(output[bin]) << std::endl;
1051  // }
1052  // }
STL namespace.