pktools  2.6.3
Processing Kernel for geospatial data
pkstat.cc
1 /**********************************************************************
2 pkstat.cc: program to calculate basic statistics from raster dataset
3 Copyright (C) 2008-2015 Pieter Kempeneers
4 
5 This file is part of pktools
6 
7 pktools is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
11 
12 pktools is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16 
17 You should have received a copy of the GNU General Public License
18 along with pktools. If not, see <http://www.gnu.org/licenses/>.
19 ***********************************************************************/
20 #include <iostream>
21 #include <fstream>
22 #include <math.h>
23 #include "base/Optionpk.h"
24 #include "algorithms/StatFactory.h"
25 #include "algorithms/ImgRegression.h"
26 using namespace std;
27 
28 int main(int argc, char *argv[])
29 {
30  Optionpk<string> input_opt("i","input","name of the input raster dataset");
31  Optionpk<unsigned short> band_opt("b","band","band(s) on which to calculate statistics",0);
32  Optionpk<bool> filename_opt("f", "filename", "Shows image filename ", false);
33  Optionpk<bool> stat_opt("stats", "statistics", "Shows basic statistics (min,max, mean and stdDev of the raster datasets)", false);
34  Optionpk<double> ulx_opt("ulx", "ulx", "Upper left x value bounding box");
35  Optionpk<double> uly_opt("uly", "uly", "Upper left y value bounding box");
36  Optionpk<double> lrx_opt("lrx", "lrx", "Lower right x value bounding box");
37  Optionpk<double> lry_opt("lry", "lry", "Lower right y value bounding box");
38  Optionpk<double> nodata_opt("nodata","nodata","Set nodata value(s)");
39  Optionpk<short> down_opt("down", "down", "Down sampling factor (for raster sample datasets only). Can be used to create grid points", 1);
40  Optionpk<unsigned int> random_opt("rnd", "rnd", "generate random numbers", 0);
41  Optionpk<double> scale_opt("scale", "scale", "Scale(s) for reading input image(s)");
42  Optionpk<double> offset_opt("offset", "offset", "Offset(s) for reading input image(s)");
43 
44  // Optionpk<bool> transpose_opt("t","transpose","transpose output",false);
45  // Optionpk<std::string> randdist_opt("dist", "dist", "distribution for generating random numbers, see http://www.gn/software/gsl/manual/gsl-ref_toc.html#TOC320 (only uniform and Gaussian supported yet)", "gaussian");
46  // Optionpk<double> randa_opt("rnda", "rnda", "first parameter for random distribution (mean value in case of Gaussian)", 0);
47  // Optionpk<double> randb_opt("rndb", "rndb", "second parameter for random distribution (standard deviation in case of Gaussian)", 1);
48  Optionpk<bool> mean_opt("mean","mean","calculate mean",false);
49  Optionpk<bool> median_opt("median","median","calculate median",false);
50  Optionpk<bool> var_opt("var","var","calculate variance",false);
51  Optionpk<bool> skewness_opt("skew","skewness","calculate skewness",false);
52  Optionpk<bool> kurtosis_opt("kurt","kurtosis","calculate kurtosis",false);
53  Optionpk<bool> stdev_opt("stdev","stdev","calculate standard deviation",false);
54  Optionpk<bool> sum_opt("sum","sum","calculate sum of column",false);
55  Optionpk<bool> minmax_opt("mm","minmax","calculate minimum and maximum value",false);
56  Optionpk<bool> min_opt("min","min","calculate minimum value",false);
57  Optionpk<bool> max_opt("max","max","calculate maximum value",false);
58  Optionpk<double> src_min_opt("src_min","src_min","start reading source from this minimum value");
59  Optionpk<double> src_max_opt("src_max","src_max","stop reading source from this maximum value");
60  Optionpk<bool> histogram_opt("hist","hist","calculate histogram",false);
61  Optionpk<bool> histogram2d_opt("hist2d","hist2d","calculate 2-dimensional histogram based on two images",false);
62  Optionpk<short> nbin_opt("nbin","nbin","number of bins to calculate histogram");
63  Optionpk<bool> relative_opt("rel","relative","use percentiles for histogram to calculate histogram",false);
64  Optionpk<bool> kde_opt("kde","kde","Use Kernel density estimation when producing histogram. The standard deviation is estimated based on Silverman's rule of thumb",false);
65  Optionpk<bool> correlation_opt("cor","correlation","calculate Pearson produc-moment correlation coefficient between two raster datasets (defined by -c <col1> -c <col2>)",false);
66  Optionpk<bool> rmse_opt("rmse","rmse","calculate root mean square error between two raster datasets",false);
67  Optionpk<bool> reg_opt("reg","regression","calculate linear regression between two raster datasets and get correlation coefficient",false);
68  Optionpk<bool> regerr_opt("regerr","regerr","calculate linear regression between two raster datasets and get root mean square error",false);
69  Optionpk<bool> preg_opt("preg","preg","calculate perpendicular regression between two raster datasets and get correlation coefficient",false);
70  Optionpk<short> verbose_opt("v", "verbose", "verbose mode when positive", 0,2);
71  ulx_opt.setHide(1);
72  uly_opt.setHide(1);
73  lrx_opt.setHide(1);
74  lry_opt.setHide(1);
75  down_opt.setHide(1);
76  random_opt.setHide(1);
77  scale_opt.setHide(1);
78  offset_opt.setHide(1);
79  src_min_opt.setHide(1);
80  src_max_opt.setHide(1);
81  kde_opt.setHide(1);
82 
83  // range_opt.setHide(1);
84  // transpose_opt.setHide(1);
85 
86  bool doProcess;//stop process when program was invoked with help option (-h --help)
87  try{
88  //mandatory options
89  doProcess=input_opt.retrieveOption(argc,argv);
90  //optional options
91  band_opt.retrieveOption(argc,argv);
92  filename_opt.retrieveOption(argc,argv);
93  stat_opt.retrieveOption(argc,argv);
94  nodata_opt.retrieveOption(argc,argv);
95  mean_opt.retrieveOption(argc,argv);
96  median_opt.retrieveOption(argc,argv);
97  var_opt.retrieveOption(argc,argv);
98  stdev_opt.retrieveOption(argc,argv);
99  minmax_opt.retrieveOption(argc,argv);
100  min_opt.retrieveOption(argc,argv);
101  max_opt.retrieveOption(argc,argv);
102  histogram_opt.retrieveOption(argc,argv);
103  nbin_opt.retrieveOption(argc,argv);
104  relative_opt.retrieveOption(argc,argv);
105  histogram2d_opt.retrieveOption(argc,argv);
106  correlation_opt.retrieveOption(argc,argv);
107  rmse_opt.retrieveOption(argc,argv);
108  reg_opt.retrieveOption(argc,argv);
109  regerr_opt.retrieveOption(argc,argv);
110  preg_opt.retrieveOption(argc,argv);
111  //advanced options
112  ulx_opt.retrieveOption(argc,argv);
113  uly_opt.retrieveOption(argc,argv);
114  lrx_opt.retrieveOption(argc,argv);
115  lry_opt.retrieveOption(argc,argv);
116  down_opt.retrieveOption(argc,argv);
117  random_opt.retrieveOption(argc,argv);
118  scale_opt.retrieveOption(argc,argv);
119  offset_opt.retrieveOption(argc,argv);
120  src_min_opt.retrieveOption(argc,argv);
121  src_max_opt.retrieveOption(argc,argv);
122  kde_opt.retrieveOption(argc,argv);
123  verbose_opt.retrieveOption(argc,argv);
124  }
125  catch(string predefinedString){
126  std::cout << predefinedString << std::endl;
127  exit(0);
128  }
129  if(!doProcess){
130  cout << endl;
131  cout << "Usage: pkstat -i input" << endl;
132  cout << endl;
133  std::cout << "short option -h shows basic options only, use long option --help to show all options" << std::endl;
134  exit(0);//help was invoked, stop processing
135  }
136 
137  if(src_min_opt.size()){
138  while(src_min_opt.size()<band_opt.size())
139  src_min_opt.push_back(src_min_opt[0]);
140  }
141  if(src_max_opt.size()){
142  while(src_max_opt.size()<band_opt.size())
143  src_max_opt.push_back(src_max_opt[0]);
144  }
145 
146  unsigned int nbin=0;
147  double minX=0;
148  double minY=0;
149  double maxX=0;
150  double maxY=0;
151  double minValue=0;
152  double maxValue=0;
153  double meanValue=0;
154  double stdDev=0;
155 
156  const char* pszMessage;
157  void* pProgressArg=NULL;
158  GDALProgressFunc pfnProgress=GDALTermProgress;
159  double progress=0;
160  srand(time(NULL));
161 
164  std::vector<double> histogramOutput;
165  double nsample=0;
166 
167  ImgReaderGdal imgReader;
168 
169  if(scale_opt.size()){
170  while(scale_opt.size()<input_opt.size())
171  scale_opt.push_back(scale_opt[0]);
172  }
173  if(offset_opt.size()){
174  while(offset_opt.size()<input_opt.size())
175  offset_opt.push_back(offset_opt[0]);
176  }
177  if(input_opt.empty()){
178  std::cerr << "No image dataset provided (use option -i). Use --help for help information";
179  exit(0);
180  }
181  for(int ifile=0;ifile<input_opt.size();++ifile){
182  try{
183  imgReader.open(input_opt[ifile]);
184  }
185  catch(std::string errorstring){
186  std::cout << errorstring << std::endl;
187  exit(0);
188  }
189 
190  if(filename_opt[0])
191  std::cout << " --input " << input_opt[ifile] << " ";
192 
193  for(int inodata=0;inodata<nodata_opt.size();++inodata)
194  imgReader.pushNoDataValue(nodata_opt[inodata]);
195 
196  int nband=band_opt.size();
197  for(int iband=0;iband<nband;++iband){
198 
199  for(int inodata=0;inodata<nodata_opt.size();++inodata){
200  if(!inodata)
201  imgReader.GDALSetNoDataValue(nodata_opt[0],iband);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
202  }
203 
204  if(offset_opt.size()>ifile)
205  imgReader.setOffset(offset_opt[ifile],band_opt[iband]);
206  if(scale_opt.size()>ifile)
207  imgReader.setScale(scale_opt[ifile],band_opt[iband]);
208 
209  // if(stat_opt[0]||mean_opt[0]||var_opt[0]||stdev_opt[0]){
210  // assert(band_opt[iband]<imgReader.nrOfBand());
211  // GDALProgressFunc pfnProgress;
212  // void* pProgressData;
213  // GDALRasterBand* rasterBand;
214  // rasterBand=imgReader.getRasterBand(band_opt[iband]);
215  // rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev,pfnProgress,pProgressData);
216 
217  // if(mean_opt[0])
218  // std::cout << "--mean " << meanValue << " ";
219  // if(stdev_opt[0])
220  // std::cout << "--stdDev " << stdDev << " ";
221  // if(var_opt[0])
222  // std::cout << "--var " << stdDev*stdDev << " ";
223  // if(stat_opt[0])
224  // std::cout << "-min " << minValue << " -max " << maxValue << " --mean " << meanValue << " --stdDev " << stdDev << " ";
225  // }
226 
227  if(minmax_opt[0]||min_opt[0]||max_opt[0]){
228  assert(band_opt[iband]<imgReader.nrOfBand());
229 
230  if((ulx_opt.size()||uly_opt.size()||lrx_opt.size()||lry_opt.size())&&(imgReader.covers(ulx_opt[0],uly_opt[0],lrx_opt[0],lry_opt[0]))){
231  double uli,ulj,lri,lrj;
232  imgReader.geo2image(ulx_opt[0],uly_opt[0],uli,ulj);
233  imgReader.geo2image(lrx_opt[0],lry_opt[0],lri,lrj);
234  imgReader.getMinMax(static_cast<int>(uli),static_cast<int>(lri),static_cast<int>(ulj),static_cast<int>(lrj),band_opt[iband],minValue,maxValue);
235  }
236  else{
237  imgReader.getMinMax(minValue,maxValue,band_opt[iband],true);
238  }
239  if(minmax_opt[0])
240  std::cout << "-min " << minValue << " -max " << maxValue << " ";
241  else{
242  if(min_opt[0])
243  std::cout << "-min " << minValue << " ";
244  if(max_opt[0])
245  std::cout << "-max " << maxValue << " ";
246  }
247  }
248  }
249  if(histogram_opt[0]){//aggregate results from multiple inputs, but only calculate for first selected band
250  assert(band_opt[0]<imgReader.nrOfBand());
251  nbin=(nbin_opt.size())? nbin_opt[0]:0;
252 
253  imgReader.getMinMax(minValue,maxValue,band_opt[0]);
254  if(src_min_opt.size())
255  minValue=src_min_opt[0];
256  if(src_max_opt.size())
257  maxValue=src_max_opt[0];
258  if(minValue>=maxValue)
259  imgReader.getMinMax(minValue,maxValue,band_opt[0]);
260 
261  if(verbose_opt[0])
262  cout << "number of valid pixels in image: " << imgReader.getNvalid(band_opt[0]) << endl;
263 
264  nsample+=imgReader.getHistogram(histogramOutput,minValue,maxValue,nbin,band_opt[0],kde_opt[0]);
265 
266  //only output for last input file
267  if(ifile==input_opt.size()-1){
268  std::cout.precision(10);
269  for(int bin=0;bin<nbin;++bin){
270  double binValue=0;
271  if(nbin==maxValue-minValue+1)
272  binValue=minValue+bin;
273  else
274  binValue=minValue+static_cast<double>(maxValue-minValue)*(bin+0.5)/nbin;
275  std::cout << binValue << " ";
276  if(relative_opt[0]||kde_opt[0])
277  std::cout << 100.0*static_cast<double>(histogramOutput[bin])/static_cast<double>(nsample) << std::endl;
278  else
279  std::cout << static_cast<double>(histogramOutput[bin]) << std::endl;
280  }
281  }
282  }
283  if(histogram2d_opt[0]&&input_opt.size()<2){
284  assert(band_opt.size()>1);
285  imgReader.getMinMax(minX,maxX,band_opt[0]);
286  imgReader.getMinMax(minY,maxY,band_opt[1]);
287  if(src_min_opt.size()){
288  minX=src_min_opt[0];
289  minY=src_min_opt[1];
290  }
291  if(src_max_opt.size()){
292  maxX=src_max_opt[0];
293  maxY=src_max_opt[1];
294  }
295  nbin=(nbin_opt.size())? nbin_opt[0]:0;
296  if(nbin<=1){
297  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
298  if(minX>=maxX)
299  imgReader.getMinMax(minX,maxX,band_opt[0]);
300  if(minY>=maxY)
301  imgReader.getMinMax(minY,maxY,band_opt[1]);
302 
303  minValue=(minX<minY)? minX:minY;
304  maxValue=(maxX>maxY)? maxX:maxY;
305  if(verbose_opt[0])
306  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
307  nbin=maxValue-minValue+1;
308  }
309  assert(nbin>1);
310  double sigma=0;
311  //kernel density estimation as in http://en.wikipedia.org/wiki/Kernel_density_estimation
312  if(kde_opt[0]){
313  assert(band_opt[0]<imgReader.nrOfBand());
314  assert(band_opt[1]<imgReader.nrOfBand());
315  GDALProgressFunc pfnProgress;
316  void* pProgressData;
317  GDALRasterBand* rasterBand;
318  double stdDev1=0;
319  double stdDev2=0;
320  rasterBand=imgReader.getRasterBand(band_opt[0]);
321  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev1,pfnProgress,pProgressData);
322  rasterBand=imgReader.getRasterBand(band_opt[1]);
323  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev2,pfnProgress,pProgressData);
324 
325  double estimatedSize=1.0*imgReader.getNvalid(band_opt[0])/down_opt[0]/down_opt[0];
326  if(random_opt[0]>0)
327  estimatedSize*=random_opt[0]/100.0;
328  sigma=1.06*sqrt(stdDev1*stdDev2)*pow(estimatedSize,-0.2);
329  }
330  assert(nbin);
331  if(verbose_opt[0]){
332  if(sigma>0)
333  std::cout << "calculating 2d kernel density estimate with sigma " << sigma << " for bands " << band_opt[0] << " and " << band_opt[1] << std::endl;
334  else
335  std::cout << "calculating 2d histogram for bands " << band_opt[0] << " and " << band_opt[1] << std::endl;
336  std::cout << "nbin: " << nbin << std::endl;
337  }
338 
339 
340  vector< vector<double> > output;
341 
342  if(maxX<=minX)
343  imgReader.getMinMax(minX,maxX,band_opt[0]);
344  if(maxY<=minY)
345  imgReader.getMinMax(minY,maxY,band_opt[1]);
346 
347  if(maxX<=minX){
348  std::ostringstream s;
349  s<<"Error: could not calculate distribution (minX>=maxX)";
350  throw(s.str());
351  }
352  if(maxY<=minY){
353  std::ostringstream s;
354  s<<"Error: could not calculate distribution (minY>=maxY)";
355  throw(s.str());
356  }
357  output.resize(nbin);
358  for(int i=0;i<nbin;++i){
359  output[i].resize(nbin);
360  for(int j=0;j<nbin;++j)
361  output[i][j]=0;
362  }
363  int binX=0;
364  int binY=0;
365  vector<double> inputX(imgReader.nrOfCol());
366  vector<double> inputY(imgReader.nrOfCol());
367  unsigned long int nvalid=0;
368  for(int irow=0;irow<imgReader.nrOfRow();++irow){
369  if(irow%down_opt[0])
370  continue;
371  imgReader.readData(inputX,GDT_Float64,irow,band_opt[0]);
372  imgReader.readData(inputY,GDT_Float64,irow,band_opt[1]);
373  for(int icol=0;icol<imgReader.nrOfCol();++icol){
374  if(icol%down_opt[0])
375  continue;
376  if(random_opt[0]>0){
377  double p=static_cast<double>(rand())/(RAND_MAX);
378  p*=100.0;
379  if(p>random_opt[0])
380  continue;//do not select for now, go to next column
381  }
382  if(imgReader.isNoData(inputX[icol]))
383  continue;
384  if(imgReader.isNoData(inputY[icol]))
385  continue;
386  ++nvalid;
387  if(inputX[icol]>=maxX)
388  binX=nbin-1;
389  else if(inputX[icol]<=minX)
390  binX=0;
391  else
392  binX=static_cast<int>(static_cast<double>(inputX[icol]-minX)/(maxX-minX)*nbin);
393  if(inputY[icol]>=maxY)
394  binY=nbin-1;
395  else if(inputY[icol]<=minX)
396  binY=0;
397  else
398  binY=static_cast<int>(static_cast<double>(inputY[icol]-minY)/(maxY-minY)*nbin);
399  assert(binX>=0);
400  assert(binX<output.size());
401  assert(binY>=0);
402  assert(binY<output[binX].size());
403  if(sigma>0){
404  //create kde for Gaussian basis function
405  //todo: speed up by calculating first and last bin with non-zero contriubtion...
406  for(int ibinX=0;ibinX<nbin;++ibinX){
407  double centerX=minX+static_cast<double>(maxX-minX)*ibinX/nbin;
408  double pdfX=gsl_ran_gaussian_pdf(inputX[icol]-centerX, sigma);
409  for(int ibinY=0;ibinY<nbin;++ibinY){
410  //calculate \integral_ibinX^(ibinX+1)
411  double centerY=minY+static_cast<double>(maxY-minY)*ibinY/nbin;
412  double pdfY=gsl_ran_gaussian_pdf(inputY[icol]-centerY, sigma);
413  output[ibinX][binY]+=pdfX*pdfY;
414  }
415  }
416  }
417  else
418  ++output[binX][binY];
419  }
420  }
421  if(verbose_opt[0])
422  cout << "number of valid pixels: " << nvalid << endl;
423 
424  for(int binX=0;binX<nbin;++binX){
425  cout << endl;
426  for(int binY=0;binY<nbin;++binY){
427  double binValueX=0;
428  if(nbin==maxX-minX+1)
429  binValueX=minX+binX;
430  else
431  binValueX=minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
432  double binValueY=0;
433  if(nbin==maxY-minY+1)
434  binValueY=minY+binY;
435  else
436  binValueY=minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
437 
438  double value=static_cast<double>(output[binX][binY]);
439 
440  if(relative_opt[0])
441  value*=100.0/nvalid;
442 
443  cout << binValueX << " " << binValueY << " " << value << std::endl;
444  // double value=static_cast<double>(output[binX][binY])/nvalid;
445  // cout << (maxX-minX)*bin/(nbin-1)+minX << " " << (maxY-minY)*bin/(nbin-1)+minY << " " << value << std::endl;
446  }
447  }
448  }
449  if(reg_opt[0]&&input_opt.size()<2){
450  if(band_opt.size()<2)
451  continue;
452  imgreg.setDown(down_opt[0]);
453  imgreg.setThreshold(random_opt[0]);
454  double c0=0;//offset
455  double c1=1;//scale
456  double r2=imgreg.getR2(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
457  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
458  }
459  if(regerr_opt[0]&&input_opt.size()<2){
460  if(band_opt.size()<2)
461  continue;
462  imgreg.setDown(down_opt[0]);
463  imgreg.setThreshold(random_opt[0]);
464  double c0=0;//offset
465  double c1=1;//scale
466  double err=imgreg.getRMSE(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
467  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -rmse " << err << std::endl;
468  }
469  if(rmse_opt[0]&&input_opt.size()<2){
470  if(band_opt.size()<2)
471  continue;
472  imgreg.setDown(down_opt[0]);
473  imgreg.setThreshold(random_opt[0]);
474  double c0=0;//offset
475  double c1=1;//scale
476  double err=imgreg.getRMSE(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
477  std::cout << " -rmse " << err << std::endl;
478  }
479  if(preg_opt[0]&&input_opt.size()<2){
480  if(band_opt.size()<2)
481  continue;
482  imgreg.setDown(down_opt[0]);
483  imgreg.setThreshold(random_opt[0]);
484  double c0=0;//offset
485  double c1=1;//scale
486  double r2=imgreg.pgetR2(imgReader,band_opt[0],band_opt[1],c0,c1,verbose_opt[0]);
487  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
488  }
489  imgReader.close();
490  }
491  if(reg_opt[0]&&(input_opt.size()>1)){
492  imgreg.setDown(down_opt[0]);
493  imgreg.setThreshold(random_opt[0]);
494  double c0=0;//offset
495  double c1=1;//scale
496  while(band_opt.size()<input_opt.size())
497  band_opt.push_back(band_opt[0]);
498  if(src_min_opt.size()){
499  while(src_min_opt.size()<input_opt.size())
500  src_min_opt.push_back(src_min_opt[0]);
501  }
502  if(src_max_opt.size()){
503  while(src_max_opt.size()<input_opt.size())
504  src_max_opt.push_back(src_max_opt[0]);
505  }
506  ImgReaderGdal imgReader1(input_opt[0]);
507  ImgReaderGdal imgReader2(input_opt[1]);
508 
509  if(offset_opt.size())
510  imgReader1.setOffset(offset_opt[0],band_opt[0]);
511  if(scale_opt.size())
512  imgReader1.setScale(scale_opt[0],band_opt[0]);
513  if(offset_opt.size()>1)
514  imgReader2.setOffset(offset_opt[1],band_opt[1]);
515  if(scale_opt.size()>1)
516  imgReader2.setScale(scale_opt[1],band_opt[1]);
517 
518  for(int inodata=0;inodata<nodata_opt.size();++inodata){
519  if(!inodata){
520  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
521  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
522  }
523  imgReader1.pushNoDataValue(nodata_opt[inodata]);
524  imgReader2.pushNoDataValue(nodata_opt[inodata]);
525  }
526 
527  double r2=imgreg.getR2(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
528  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
529  imgReader1.close();
530  imgReader2.close();
531  }
532  if(preg_opt[0]&&(input_opt.size()>1)){
533  imgreg.setDown(down_opt[0]);
534  imgreg.setThreshold(random_opt[0]);
535  double c0=0;//offset
536  double c1=1;//scale
537  while(band_opt.size()<input_opt.size())
538  band_opt.push_back(band_opt[0]);
539  if(src_min_opt.size()){
540  while(src_min_opt.size()<input_opt.size())
541  src_min_opt.push_back(src_min_opt[0]);
542  }
543  if(src_max_opt.size()){
544  while(src_max_opt.size()<input_opt.size())
545  src_max_opt.push_back(src_max_opt[0]);
546  }
547  ImgReaderGdal imgReader1(input_opt[0]);
548  ImgReaderGdal imgReader2(input_opt[1]);
549 
550  if(offset_opt.size())
551  imgReader1.setOffset(offset_opt[0],band_opt[0]);
552  if(scale_opt.size())
553  imgReader1.setScale(scale_opt[0],band_opt[0]);
554  if(offset_opt.size()>1)
555  imgReader2.setOffset(offset_opt[1],band_opt[1]);
556  if(scale_opt.size()>1)
557  imgReader2.setScale(scale_opt[1],band_opt[1]);
558 
559  for(int inodata=0;inodata<nodata_opt.size();++inodata){
560  if(!inodata){
561  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
562  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
563  }
564  imgReader1.pushNoDataValue(nodata_opt[inodata]);
565  imgReader2.pushNoDataValue(nodata_opt[inodata]);
566  }
567 
568  double r2=imgreg.pgetR2(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
569  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -r2 " << r2 << std::endl;
570  imgReader1.close();
571  imgReader2.close();
572  }
573  if(regerr_opt[0]&&(input_opt.size()>1)){
574  imgreg.setDown(down_opt[0]);
575  imgreg.setThreshold(random_opt[0]);
576  double c0=0;//offset
577  double c1=1;//scale
578  while(band_opt.size()<input_opt.size())
579  band_opt.push_back(band_opt[0]);
580  if(src_min_opt.size()){
581  while(src_min_opt.size()<input_opt.size())
582  src_min_opt.push_back(src_min_opt[0]);
583  }
584  if(src_max_opt.size()){
585  while(src_max_opt.size()<input_opt.size())
586  src_max_opt.push_back(src_max_opt[0]);
587  }
588  ImgReaderGdal imgReader1(input_opt[0]);
589  ImgReaderGdal imgReader2(input_opt[1]);
590 
591  if(offset_opt.size())
592  imgReader1.setOffset(offset_opt[0],band_opt[0]);
593  if(scale_opt.size())
594  imgReader1.setScale(scale_opt[0],band_opt[0]);
595  if(offset_opt.size()>1)
596  imgReader2.setOffset(offset_opt[1],band_opt[1]);
597  if(scale_opt.size()>1)
598  imgReader2.setScale(scale_opt[1],band_opt[1]);
599 
600  for(int inodata=0;inodata<nodata_opt.size();++inodata){
601  if(!inodata){
602  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
603  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
604  }
605  imgReader1.pushNoDataValue(nodata_opt[inodata]);
606  imgReader2.pushNoDataValue(nodata_opt[inodata]);
607  }
608 
609  double err=imgreg.getRMSE(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
610  std::cout << "-c0 " << c0 << " -c1 " << c1 << " -rmse " << err << std::endl;
611  imgReader1.close();
612  imgReader2.close();
613  }
614  if(rmse_opt[0]&&(input_opt.size()>1)){
615  imgreg.setDown(down_opt[0]);
616  imgreg.setThreshold(random_opt[0]);
617  double c0=0;//offset
618  double c1=1;//scale
619  while(band_opt.size()<input_opt.size())
620  band_opt.push_back(band_opt[0]);
621  if(src_min_opt.size()){
622  while(src_min_opt.size()<input_opt.size())
623  src_min_opt.push_back(src_min_opt[0]);
624  }
625  if(src_max_opt.size()){
626  while(src_max_opt.size()<input_opt.size())
627  src_max_opt.push_back(src_max_opt[0]);
628  }
629  ImgReaderGdal imgReader1(input_opt[0]);
630  ImgReaderGdal imgReader2(input_opt[1]);
631 
632  if(offset_opt.size())
633  imgReader1.setOffset(offset_opt[0],band_opt[0]);
634  if(scale_opt.size())
635  imgReader1.setScale(scale_opt[0],band_opt[0]);
636  if(offset_opt.size()>1)
637  imgReader2.setOffset(offset_opt[1],band_opt[1]);
638  if(scale_opt.size()>1)
639  imgReader2.setScale(scale_opt[1],band_opt[1]);
640 
641  for(int inodata=0;inodata<nodata_opt.size();++inodata){
642  if(!inodata){
643  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
644  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
645  }
646  imgReader1.pushNoDataValue(nodata_opt[inodata]);
647  imgReader2.pushNoDataValue(nodata_opt[inodata]);
648  }
649 
650  double err=imgreg.getRMSE(imgReader1,imgReader2,c0,c1,band_opt[0],band_opt[1],verbose_opt[0]);
651  std::cout << "-rmse " << err << std::endl;
652  imgReader1.close();
653  imgReader2.close();
654  }
655  if(histogram2d_opt[0]&&(input_opt.size()>1)){
656  while(band_opt.size()<input_opt.size())
657  band_opt.push_back(band_opt[0]);
658  if(src_min_opt.size()){
659  while(src_min_opt.size()<input_opt.size())
660  src_min_opt.push_back(src_min_opt[0]);
661  }
662  if(src_max_opt.size()){
663  while(src_max_opt.size()<input_opt.size())
664  src_max_opt.push_back(src_max_opt[0]);
665  }
666  ImgReaderGdal imgReader1(input_opt[0]);
667  ImgReaderGdal imgReader2(input_opt[1]);
668 
669  if(offset_opt.size())
670  imgReader1.setOffset(offset_opt[0],band_opt[0]);
671  if(scale_opt.size())
672  imgReader1.setScale(scale_opt[0],band_opt[0]);
673  if(offset_opt.size()>1)
674  imgReader2.setOffset(offset_opt[1],band_opt[1]);
675  if(scale_opt.size()>1)
676  imgReader2.setScale(scale_opt[1],band_opt[1]);
677 
678  for(int inodata=0;inodata<nodata_opt.size();++inodata){
679  if(!inodata){
680  imgReader1.GDALSetNoDataValue(nodata_opt[0],band_opt[0]);//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
681  imgReader2.GDALSetNoDataValue(nodata_opt[0]),band_opt[1];//only single no data can be set in GDALRasterBand (used for ComputeStatistics)
682  }
683  imgReader1.pushNoDataValue(nodata_opt[inodata]);
684  imgReader2.pushNoDataValue(nodata_opt[inodata]);
685  }
686 
687  imgReader1.getMinMax(minX,maxX,band_opt[0]);
688  imgReader2.getMinMax(minY,maxY,band_opt[1]);
689 
690  if(verbose_opt[0]){
691  cout << "minX: " << minX << endl;
692  cout << "maxX: " << maxX << endl;
693  cout << "minY: " << minY << endl;
694  cout << "maxY: " << maxY << endl;
695  }
696 
697  if(src_min_opt.size()){
698  minX=src_min_opt[0];
699  minY=src_min_opt[1];
700  }
701  if(src_max_opt.size()){
702  maxX=src_max_opt[0];
703  maxY=src_max_opt[1];
704  }
705 
706  nbin=(nbin_opt.size())? nbin_opt[0]:0;
707  if(nbin<=1){
708  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
709  // imgReader1.getMinMax(minX,maxX,band_opt[0]);
710  // imgReader2.getMinMax(minY,maxY,band_opt[0]);
711  if(minX>=maxX)
712  imgReader1.getMinMax(minX,maxX,band_opt[0]);
713  if(minY>=maxY)
714  imgReader2.getMinMax(minY,maxY,band_opt[1]);
715 
716  minValue=(minX<minY)? minX:minY;
717  maxValue=(maxX>maxY)? maxX:maxY;
718  if(verbose_opt[0])
719  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
720  nbin=maxValue-minValue+1;
721  }
722  assert(nbin>1);
723  double sigma=0;
724  //kernel density estimation as in http://en.wikipedia.org/wiki/Kernel_density_estimation
725  if(kde_opt[0]){
726  GDALProgressFunc pfnProgress;
727  void* pProgressData;
728  GDALRasterBand* rasterBand;
729  double stdDev1=0;
730  double stdDev2=0;
731  rasterBand=imgReader1.getRasterBand(band_opt[0]);
732  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev1,pfnProgress,pProgressData);
733  rasterBand=imgReader2.getRasterBand(band_opt[0]);
734  rasterBand->ComputeStatistics(0,&minValue,&maxValue,&meanValue,&stdDev2,pfnProgress,pProgressData);
735 
736  //todo: think of smarter way how to estimate size (nodata!)
737  double estimatedSize=1.0*imgReader.getNvalid(band_opt[0])/down_opt[0]/down_opt[0];
738  if(random_opt[0]>0)
739  estimatedSize*=random_opt[0]/100.0;
740  sigma=1.06*sqrt(stdDev1*stdDev2)*pow(estimatedSize,-0.2);
741  }
742  assert(nbin);
743  if(verbose_opt[0]){
744  if(sigma>0)
745  std::cout << "calculating 2d kernel density estimate with sigma " << sigma << " for datasets " << input_opt[0] << " and " << input_opt[1] << std::endl;
746  else
747  std::cout << "calculating 2d histogram for datasets " << input_opt[0] << " and " << input_opt[1] << std::endl;
748  std::cout << "nbin: " << nbin << std::endl;
749  }
750 
751  vector< vector<double> > output;
752 
753  if(maxX<=minX)
754  imgReader1.getMinMax(minX,maxX,band_opt[0]);
755  if(maxY<=minY)
756  imgReader2.getMinMax(minY,maxY,band_opt[1]);
757 
758  if(maxX<=minX){
759  std::ostringstream s;
760  s<<"Error: could not calculate distribution (minX>=maxX)";
761  throw(s.str());
762  }
763  if(maxY<=minY){
764  std::ostringstream s;
765  s<<"Error: could not calculate distribution (minY>=maxY)";
766  throw(s.str());
767  }
768  if(verbose_opt[0]){
769  cout << "minX: " << minX << endl;
770  cout << "maxX: " << maxX << endl;
771  cout << "minY: " << minY << endl;
772  cout << "maxY: " << maxY << endl;
773  }
774  output.resize(nbin);
775  for(int i=0;i<nbin;++i){
776  output[i].resize(nbin);
777  for(int j=0;j<nbin;++j)
778  output[i][j]=0;
779  }
780  int binX=0;
781  int binY=0;
782  vector<double> inputX(imgReader1.nrOfCol());
783  vector<double> inputY(imgReader2.nrOfCol());
784  double nvalid=0;
785  double geoX=0;
786  double geoY=0;
787  double icol1=0;
788  double irow1=0;
789  double icol2=0;
790  double irow2=0;
791  for(int irow=0;irow<imgReader1.nrOfRow();++irow){
792  if(irow%down_opt[0])
793  continue;
794  irow1=irow;
795  imgReader1.image2geo(icol1,irow1,geoX,geoY);
796  imgReader2.geo2image(geoX,geoY,icol2,irow2);
797  irow2=static_cast<int>(irow2);
798  imgReader1.readData(inputX,GDT_Float64,irow1,band_opt[0]);
799  imgReader2.readData(inputY,GDT_Float64,irow2,band_opt[1]);
800  for(int icol=0;icol<imgReader.nrOfCol();++icol){
801  if(icol%down_opt[0])
802  continue;
803  icol1=icol;
804  if(random_opt[0]>0){
805  double p=static_cast<double>(rand())/(RAND_MAX);
806  p*=100.0;
807  if(p>random_opt[0])
808  continue;//do not select for now, go to next column
809  }
810  if(imgReader1.isNoData(inputX[icol]))
811  continue;
812  imgReader1.image2geo(icol1,irow1,geoX,geoY);
813  imgReader2.geo2image(geoX,geoY,icol2,irow2);
814  icol2=static_cast<int>(icol2);
815  if(imgReader2.isNoData(inputY[icol2]))
816  continue;
817  // ++nvalid;
818  if(inputX[icol1]>=maxX)
819  binX=nbin-1;
820  else if(inputX[icol]<=minX)
821  binX=0;
822  else
823  binX=static_cast<int>(static_cast<double>(inputX[icol1]-minX)/(maxX-minX)*nbin);
824  if(inputY[icol2]>=maxY)
825  binY=nbin-1;
826  else if(inputY[icol2]<=minY)
827  binY=0;
828  else
829  binY=static_cast<int>(static_cast<double>(inputY[icol2]-minY)/(maxY-minY)*nbin);
830  assert(binX>=0);
831  assert(binX<output.size());
832  assert(binY>=0);
833  assert(binY<output[binX].size());
834  if(sigma>0){
835  //create kde for Gaussian basis function
836  //todo: speed up by calculating first and last bin with non-zero contriubtion...
837  for(int ibinX=0;ibinX<nbin;++ibinX){
838  double centerX=minX+static_cast<double>(maxX-minX)*ibinX/nbin;
839  double pdfX=gsl_ran_gaussian_pdf(inputX[icol1]-centerX, sigma);
840  for(int ibinY=0;ibinY<nbin;++ibinY){
841  //calculate \integral_ibinX^(ibinX+1)
842  double centerY=minY+static_cast<double>(maxY-minY)*ibinY/nbin;
843  double pdfY=gsl_ran_gaussian_pdf(inputY[icol2]-centerY, sigma);
844  output[ibinX][binY]+=pdfX*pdfY;
845  nvalid+=pdfX*pdfY;
846  }
847  }
848  }
849  else{
850  ++output[binX][binY];
851  ++nvalid;
852  }
853  }
854  }
855  if(verbose_opt[0])
856  cout << "number of valid pixels: " << nvalid << endl;
857  for(int binX=0;binX<nbin;++binX){
858  cout << endl;
859  for(int binY=0;binY<nbin;++binY){
860  double binValueX=0;
861  if(nbin==maxX-minX+1)
862  binValueX=minX+binX;
863  else
864  binValueX=minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
865  double binValueY=0;
866  if(nbin==maxY-minY+1)
867  binValueY=minY+binY;
868  else
869  binValueY=minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
870  double value=static_cast<double>(output[binX][binY]);
871 
872  if(relative_opt[0]||kde_opt[0])
873  value*=100.0/nvalid;
874 
875  cout << binValueX << " " << binValueY << " " << value << std::endl;
876  // double value=static_cast<double>(output[binX][binY])/nvalid;
877  // cout << (maxX-minX)*bin/(nbin-1)+minX << " " << (maxY-minY)*bin/(nbin-1)+minY << " " << value << std::endl;
878  }
879  }
880  imgReader1.close();
881  imgReader2.close();
882  }
883 
884  if(!histogram_opt[0]||histogram2d_opt[0])
885  std::cout << std::endl;
886 }
887 
888 // int nband=(band_opt.size()) ? band_opt.size() : imgReader.nrOfBand();
889 
890 // const char* pszMessage;
891 // void* pProgressArg=NULL;
892 // GDALProgressFunc pfnProgress=GDALTermProgress;
893 // double progress=0;
894 // srand(time(NULL));
895 
896 
897 // statfactory::StatFactory stat;
898 // imgregression::ImgRegression imgreg;
899 
900 // pfnProgress(progress,pszMessage,pProgressArg);
901 // for(irow=0;irow<classReader.nrOfRow();++irow){
902 // if(irow%down_opt[0])
903 // continue;
904 // // classReader.readData(classBuffer,GDT_Int32,irow);
905 // classReader.readData(classBuffer,GDT_Float64,irow);
906 // double x,y;//geo coordinates
907 // double iimg,jimg;//image coordinates in img image
908 // for(icol=0;icol<classReader.nrOfCol();++icol){
909 // if(icol%down_opt[0])
910  // continue;
911 
912 
913  // if(rand_opt[0]>0){
914  // gsl_rng* r=stat.getRandomGenerator(time(NULL));
915  // //todo: init random number generator using time...
916  // if(verbose_opt[0])
917  // std::cout << "generating " << rand_opt[0] << " random numbers: " << std::endl;
918  // for(unsigned int i=0;i<rand_opt[0];++i)
919  // std::cout << i << " " << stat.getRandomValue(r,randdist_opt[0],randa_opt[0],randb_opt[0]) << std::endl;
920  // }
921 
922  // imgreg.setDown(down_opt[0]);
923  // imgreg.setThreshold(threshold_opt[0]);
924  // double c0=0;//offset
925  // double c1=1;//scale
926  // double err=uncertNodata_opt[0];//start with high initial value in case we do not have first ob err=imgreg.getRMSE(imgReaderModel1,imgReader,c0,c1,verbose_opt[0]);
927 
928  // int nband=band_opt.size();
929  // if(band_opt[0]<0)
930  // nband=imgReader.nrOfBand();
931  // for(int iband=0;iband<nband;++iband){
932  // unsigned short band_opt[iband]=(band_opt[0]<0)? iband : band_opt[iband];
933 
934  // if(minmax_opt[0]||min_opt[0]||max_opt[0]){
935  // assert(band_opt[iband]<imgReader.nrOfBand());
936  // if((ulx_opt.size()||uly_opt.size()||lrx_opt.size()||lry_opt.size())&&(imgReader.covers(ulx_opt[0],uly_opt[0],lrx_opt[0],lry_opt[0]))){
937  // double uli,ulj,lri,lrj;
938  // imgReader.geo2image(ulx_opt[0],uly_opt[0],uli,ulj);
939  // imgReader.geo2image(lrx_opt[0],lry_opt[0],lri,lrj);
940  // imgReader.getMinMax(static_cast<int>(uli),static_cast<int>(lri),static_cast<int>(ulj),static_cast<int>(lrj),band_opt[iband],minValue,maxValue);
941  // }
942  // else
943  // imgReader.getMinMax(minValue,maxValue,band_opt[iband],true);
944  // if(minmax_opt[0])
945  // std::cout << "-min " << minValue << " -max " << maxValue << " ";
946  // else{
947  // if(min_opt[0])
948  // std::cout << "-min " << minValue << " ";
949  // if(max_opt[0])
950  // std::cout << "-max " << maxValue << " ";
951  // }
952  // }
953  // }
954  // if(relative_opt[0])
955  // hist_opt[0]=true;
956  // if(hist_opt[0]){
957  // assert(band_opt[0]<imgReader.nrOfBand());
958  // unsigned int nbin=(nbin_opt.size())? nbin_opt[0]:0;
959  // std::vector<unsigned long int> output;
960  // minValue=0;
961  // maxValue=0;
962  // //todo: optimize such that getMinMax is only called once...
963  // imgReader.getMinMax(minValue,maxValue,band_opt[0]);
964 
965  // if(src_min_opt.size())
966  // minValue=src_min_opt[0];
967  // if(src_max_opt.size())
968  // maxValue=src_max_opt[0];
969  // unsigned long int nsample=imgReader.getHistogram(output,minValue,maxValue,nbin,band_opt[0]);
970  // std::cout.precision(10);
971  // for(int bin=0;bin<nbin;++bin){
972  // double binValue=0;
973  // if(nbin==maxValue-minValue+1)
974  // binValue=minValue+bin;
975  // else
976  // binValue=minValue+static_cast<double>(maxValue-minValue)*(bin+0.5)/nbin;
977  // std::cout << binValue << " ";
978  // if(relative_opt[0])
979  // std::cout << 100.0*static_cast<double>(output[bin])/static_cast<double>(nsample) << std::endl;
980  // else
981  // std::cout << static_cast<double>(output[bin]) << std::endl;
982  // }
983  // }