pktools  2.6.3
Processing Kernel for geospatial data
pkstatascii.cc
1 /**********************************************************************
2 pkstatascii.cc: program to calculate basic statistics from text file
3 Copyright (C) 2008-2014 Pieter Kempeneers
4 
5 This file is part of pktools
6 
7 pktools is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
11 
12 pktools is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16 
17 You should have received a copy of the GNU General Public License
18 along with pktools. If not, see <http://www.gnu.org/licenses/>.
19 ***********************************************************************/
20 #include <iostream>
21 #include <fstream>
22 #include <vector>
23 #include <math.h>
24 #include "base/Optionpk.h"
25 #include "fileclasses/FileReaderAscii.h"
26 #include "algorithms/StatFactory.h"
27 using namespace std;
28 
29 int main(int argc, char *argv[])
30 {
31  Optionpk<string> input_opt("i","input","name of the input text file");
32  Optionpk<char> fs_opt("fs","fs","field separator.",' ');
33  Optionpk<char> comment_opt("comment","comment","comment character",'#');
34  Optionpk<bool> output_opt("o","output","output the selected columns",false);
35  Optionpk<bool> transpose_opt("t","transpose","transpose input ascii vector (use in combination with --output)",false);
36  Optionpk<int> col_opt("c", "column", "column nr, starting from 0", 0);
37  Optionpk<int> range_opt("r", "range", "rows to start/end reading. Use -r 1 -r 10 to read first 10 rows where first row is header. Use 0 to read all rows with no header.", 0);
38  Optionpk<bool> size_opt("size","size","sample size",false);
39  Optionpk<unsigned int> rand_opt("rnd", "rnd", "generate random numbers", 0);
40  Optionpk<std::string> randdist_opt("dist", "dist", "distribution for generating random numbers, see http://www.gn/software/gsl/manual/gsl-ref_toc.html#TOC320 (only uniform and Gaussian supported yet)", "gaussian");
41  Optionpk<double> randa_opt("rnda", "rnda", "first parameter for random distribution (mean value in case of Gaussian)", 0);
42  Optionpk<double> randb_opt("rndb", "rndb", "second parameter for random distribution (standard deviation in case of Gaussian)", 1);
43  Optionpk<bool> mean_opt("mean","mean","calculate median",false);
44  Optionpk<bool> median_opt("median","median","calculate median",false);
45  Optionpk<bool> var_opt("var","var","calculate variance",false);
46  Optionpk<bool> skewness_opt("skew","skewness","calculate skewness",false);
47  Optionpk<bool> kurtosis_opt("kurt","kurtosis","calculate kurtosis",false);
48  Optionpk<bool> stdev_opt("stdev","stdev","calculate standard deviation",false);
49  Optionpk<bool> sum_opt("sum","sum","calculate sum of column",false);
50  Optionpk<bool> minmax_opt("mm","minmax","calculate minimum and maximum value",false);
51  Optionpk<bool> min_opt("min","min","calculate minimum value",false);
52  Optionpk<bool> max_opt("max","max","calculate maximum value",false);
53  Optionpk<double> src_min_opt("src_min","src_min","start reading source from this minimum value");
54  Optionpk<double> src_max_opt("src_max","src_max","stop reading source from this maximum value");
55  Optionpk<bool> histogram_opt("hist","hist","calculate histogram",false);
56  Optionpk<bool> histogram2d_opt("hist2d","hist2d","calculate 2-dimensional histogram based on two columns",false);
57  Optionpk<short> nbin_opt("nbin","nbin","number of bins to calculate histogram");
58  Optionpk<bool> relative_opt("rel","relative","use percentiles for histogram to calculate histogram",false);
59  Optionpk<bool> kde_opt("kde","kde","Use Kernel density estimation when producing histogram. The standard deviation is estimated based on Silverman's rule of thumb",false);
60  Optionpk<bool> correlation_opt("cor","correlation","calculate Pearson produc-moment correlation coefficient between two columns (defined by -c <col1> -c <col2>",false);
61  Optionpk<bool> rmse_opt("rmse","rmse","calculate root mean square error between two columns (defined by -c <col1> -c <col2>",false);
62  Optionpk<bool> reg_opt("reg","regression","calculate linear regression between two columns and get correlation coefficient (defined by -c <col1> -c <col2>",false);
63  Optionpk<bool> regerr_opt("regerr","regerr","calculate linear regression between two columns and get root mean square error (defined by -c <col1> -c <col2>",false);
64  Optionpk<short> verbose_opt("v", "verbose", "verbose mode when positive", 0,2);
65 
66  src_min_opt.setHide(1);
67  src_max_opt.setHide(1);
68  fs_opt.setHide(1);
69  range_opt.setHide(1);
70  output_opt.setHide(1);
71  transpose_opt.setHide(1);
72  comment_opt.setHide(1);
73 
74  bool doProcess;//stop process when program was invoked with help option (-h --help)
75  try{
76  //mandatory options
77  doProcess=input_opt.retrieveOption(argc,argv);
78  col_opt.retrieveOption(argc,argv);
79  //optional options
80  size_opt.retrieveOption(argc,argv);
81  rand_opt.retrieveOption(argc,argv);
82  randdist_opt.retrieveOption(argc,argv);
83  randa_opt.retrieveOption(argc,argv);
84  randb_opt.retrieveOption(argc,argv);
85  mean_opt.retrieveOption(argc,argv);
86  median_opt.retrieveOption(argc,argv);
87  var_opt.retrieveOption(argc,argv);
88  stdev_opt.retrieveOption(argc,argv);
89  skewness_opt.retrieveOption(argc,argv);
90  kurtosis_opt.retrieveOption(argc,argv);
91  sum_opt.retrieveOption(argc,argv);
92  minmax_opt.retrieveOption(argc,argv);
93  min_opt.retrieveOption(argc,argv);
94  max_opt.retrieveOption(argc,argv);
95  histogram_opt.retrieveOption(argc,argv);
96  nbin_opt.retrieveOption(argc,argv);
97  relative_opt.retrieveOption(argc,argv);
98  kde_opt.retrieveOption(argc,argv);
99  histogram2d_opt.retrieveOption(argc,argv);
100  correlation_opt.retrieveOption(argc,argv);
101  rmse_opt.retrieveOption(argc,argv);
102  reg_opt.retrieveOption(argc,argv);
103  regerr_opt.retrieveOption(argc,argv);
104  //advanced options
105  src_min_opt.retrieveOption(argc,argv);
106  src_max_opt.retrieveOption(argc,argv);
107  fs_opt.retrieveOption(argc,argv);
108  range_opt.retrieveOption(argc,argv);
109  output_opt.retrieveOption(argc,argv);
110  transpose_opt.retrieveOption(argc,argv);
111  comment_opt.retrieveOption(argc,argv);
112  verbose_opt.retrieveOption(argc,argv);
113  }
114  catch(string predefinedString){
115  std::cout << predefinedString << std::endl;
116  exit(0);
117  }
118  if(!doProcess){
119  cout << endl;
120  cout << "Usage: pkstatascii -i input [-c column]*" << endl;
121  cout << endl;
122  std::cout << "short option -h shows basic options only, use long option --help to show all options" << std::endl;
123  exit(0);//help was invoked, stop processing
124  }
125 
126  if(src_min_opt.size()){
127  while(src_min_opt.size()<col_opt.size())
128  src_min_opt.push_back(src_min_opt[0]);
129  }
130  if(src_max_opt.size()){
131  while(src_max_opt.size()<col_opt.size())
132  src_max_opt.push_back(src_max_opt[0]);
133  }
135  if(rand_opt[0]>0){
136  gsl_rng* r=stat.getRandomGenerator(time(NULL));
137  //todo: init random number generator using time...
138  if(verbose_opt[0])
139  std::cout << "generating " << rand_opt[0] << " random numbers: " << std::endl;
140  for(unsigned int i=0;i<rand_opt[0];++i)
141  std::cout << i << " " << stat.getRandomValue(r,randdist_opt[0],randa_opt[0],randb_opt[0]) << std::endl;
142  }
143  vector< vector<double> > dataVector(col_opt.size());
144  vector< vector<double> > statVector(col_opt.size());
145 
146  if(!input_opt.size())
147  exit(0);
148  FileReaderAscii asciiReader(input_opt[0]);
149  asciiReader.setFieldSeparator(fs_opt[0]);
150  asciiReader.setComment(comment_opt[0]);
151  asciiReader.setMinRow(range_opt[0]);
152  if(range_opt.size()>1)
153  asciiReader.setMaxRow(range_opt[1]);
154  asciiReader.readData(dataVector,col_opt);
155  assert(dataVector.size());
156  double minValue=0;
157  double maxValue=0;
158  unsigned int nbin=0;
159  if(nbin_opt.size())
160  nbin=nbin_opt[0];
161  if(histogram_opt[0]){
162  stat.minmax(dataVector[0],dataVector[0].begin(),dataVector[0].end(),minValue,maxValue);
163  if(src_min_opt.size())
164  minValue=src_min_opt[0];
165  if(src_max_opt.size())
166  maxValue=src_max_opt[0];
167  if(nbin<1){
168  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
169  nbin=maxValue-minValue+1;
170  }
171  }
172  double minX=0;
173  double minY=0;
174  double maxX=0;
175  double maxY=0;
176  if(histogram2d_opt[0]){
177  assert(col_opt.size()==2);
178  if(nbin<1){
179  std::cerr << "Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
180  stat.minmax(dataVector[0],dataVector[0].begin(),dataVector[0].end(),minX,maxX);
181  stat.minmax(dataVector[1],dataVector[1].begin(),dataVector[1].end(),minY,maxY);
182  if(src_min_opt.size())
183  minX=src_min_opt[0];
184  if(src_min_opt.size()>1)
185  minY=src_min_opt[1];
186  if(src_max_opt.size())
187  maxX=src_max_opt[0];
188  if(src_max_opt.size()>1)
189  maxY=src_max_opt[1];
190  minValue=(minX<minY)? minX:minY;
191  maxValue=(maxX>maxY)? maxX:maxY;
192  if(verbose_opt[0])
193  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
194  nbin=maxValue-minValue+1;
195  }
196  }
197  for(int icol=0;icol<col_opt.size();++icol){
198  if(!dataVector[icol].size()){
199  std::cerr << "Warning: dataVector[" << icol << "] is empty" << std::endl;
200  continue;
201  }
202  if(size_opt[0])
203  cout << "sample size column " << col_opt[icol] << ": " << dataVector[icol].size() << endl;
204  if(mean_opt[0])
205  cout << "mean value column " << col_opt[icol] << ": " << stat.mean(dataVector[icol]) << endl;
206  if(var_opt[0])
207  cout << "variance value column " << col_opt[icol] << ": " << stat.var(dataVector[icol]) << endl;
208  if(stdev_opt[0])
209  cout << "standard deviation column " << col_opt[icol] << ": " << sqrt(stat.var(dataVector[icol])) << endl;
210  if(skewness_opt[0])
211  cout << "skewness value column " << col_opt[icol] << ": " << stat.skewness(dataVector[icol]) << endl;
212  if(kurtosis_opt[0])
213  cout << "kurtosis value column " << col_opt[icol] << ": " << stat.kurtosis(dataVector[icol]) << endl;
214  if(sum_opt[0]){
215  cout << setprecision(2);
216  cout << fixed << "sum column " << col_opt[icol] << ": " << (stat.sum(dataVector[icol])) << endl;
217  }
218  if(median_opt[0])
219  cout << "median value column " << col_opt[icol] << ": " << stat.median(dataVector[icol]) << endl;
220  if(minmax_opt[0]){
221  cout << "min value column " << col_opt[icol] << ": " << stat.mymin(dataVector[icol]) << endl;
222  cout << "max value column " << col_opt[icol] << ": " << stat.mymax(dataVector[icol]) << endl;
223  }
224  if(min_opt[0])
225  cout << "min value column " << col_opt[icol] << ": " << stat.mymin(dataVector[icol]) << endl;
226  if(max_opt[0])
227  cout << "max value column " << col_opt[icol] << ": " << stat.mymax(dataVector[icol]) << endl;
228  if(histogram_opt[0]){
229  //todo: support kernel density function and estimate sigma as in practical estimate of the bandwith in http://en.wikipedia.org/wiki/Kernel_density_estimation
230  double sigma=0;
231  if(kde_opt[0]){//.size()){
232  // if(kde_opt[0]>0)
233  // sigma=kde_opt[0];
234  // else
235  sigma=1.06*sqrt(stat.var(dataVector[icol]))*pow(dataVector[icol].size(),-0.2);
236  }
237  assert(nbin);
238  if(verbose_opt[0]){
239  if(sigma>0)
240  std::cout << "calculating kernel density estimate with sigma " << sigma << " for col " << icol << std::endl;
241  else
242  std::cout << "calculating histogram for col " << icol << std::endl;
243  }
244  //test
245  // cout << "debug0" << endl;
246  // cout << "dataVector.size(): " << dataVector.size() << endl;
247  // cout << "statVector.size(): " << statVector.size() << endl;
248 
249  // double theMinValue=0;
250  // double theMaxValue=0;
251 
252  // stat.minmax(dataVector[icol],dataVector[icol].begin(),dataVector[icol].end(),theMinValue,theMaxValue);
253  // if(minValue<maxValue&&minValue>theMinValue)
254  // theMinValue=minValue;
255  // if(minValue<maxValue&&maxValue<theMaxValue)
256  // theMaxValue=maxValue;
257 
258  // //todo: check...
259  // minValue=theMinValue;
260  // maxValue=theMaxValue;
261 
262  // if(maxValue<=minValue){
263  // std::ostringstream s;
264  // s<<"Error: could not calculate distribution (min>=max)";
265  // throw(s.str());
266  // }
267  // assert(nbin);
268  // assert(dataVector[icol].size());
269  // if(statVector[icol].size()!=nbin){
270  // statVector[icol].resize(nbin);
271  // for(int i=0;i<nbin;statVector[icol][i++]=0);
272  // }
273  // typename std::vector<double>::const_iterator it;
274  // for(it=dataVector[icol].begin();it!=dataVector[icol].end();++it){
275  // if(*it<minValue)
276  // continue;
277  // if(*it>maxValue)
278  // continue;
279  // if(stat.isNoData(*it))
280  // continue;
281  // int theBin=0;
282  // if(*it==maxValue)
283  // theBin=nbin-1;
284  // else if(*it>minValue && *it<maxValue)
285  // theBin=static_cast<int>(static_cast<double>((nbin-1)*(*it)-minValue)/(maxValue-minValue));
286  // assert(theBin<statVector[icol].size());
287  // ++statVector[icol][theBin];
288  // // if(*it==maxValue)
289  // // ++statVector[icol][nbin-1];
290  // // else if(*it>=minValue && *it<maxValue)
291  // // ++statVector[icol][static_cast<int>(static_cast<double>((*it)-minValue)/(maxValue-minValue)*nbin)];
292  // }
293 
294  // exit(0);
295  //end test
296 
297  stat.distribution(dataVector[icol],dataVector[icol].begin(),dataVector[icol].end(),statVector[icol],nbin,minValue,maxValue,sigma);
298  if(verbose_opt[0])
299  std::cout << "min and max values: " << minValue << ", " << maxValue << std::endl;
300  }
301  }
302  if(correlation_opt[0]){
303  assert(dataVector.size()==2);
304  cout << "correlation between columns " << col_opt[0] << " and " << col_opt[1] << ": " << stat.correlation(dataVector[0],dataVector[1]) << endl;
305  }
306  if(rmse_opt[0]){
307  assert(dataVector.size()==2);
308  cout << "root mean square error between columns " << col_opt[0] << " and " << col_opt[1] << ": " << stat.rmse(dataVector[0],dataVector[1]) << endl;
309  }
310  if(reg_opt[0]){
311  assert(dataVector.size()==2);
312  double c0=0;
313  double c1=0;
314  double r2=stat.linear_regression(dataVector[0],dataVector[1],c0,c1);
315  cout << "linear regression between columns: " << col_opt[0] << " and " << col_opt[1] << ": " << c0 << "+" << c1 << " * x " << " with R^2 (square correlation coefficient): " << r2 << endl;
316  }
317  if(regerr_opt[0]){
318  assert(dataVector.size()==2);
319  double c0=0;
320  double c1=0;
321  double err=stat.linear_regression_err(dataVector[0],dataVector[1],c0,c1);
322  if(verbose_opt[0])
323  cout << "linear regression between columns: " << col_opt[0] << " and " << col_opt[1] << ": " << c0 << "+" << c1 << " * x " << " with rmse: " << err << endl;
324  else
325  cout << c0 << " " << c1 << " " << err << endl;
326  }
327  if(histogram_opt[0]){
328  for(int irow=0;irow<statVector.begin()->size();++irow){
329  double binValue=0;
330  if(nbin==maxValue-minValue+1)
331  binValue=minValue+irow;
332  else
333  binValue=minValue+static_cast<double>(maxValue-minValue)*(irow+0.5)/nbin;
334  std::cout << binValue << " ";
335  // std::cout << minValue+static_cast<double>(maxValue-minValue)*(irow+0.5)/nbin << " ";
336  for(int icol=0;icol<col_opt.size();++icol){
337  if(relative_opt[0])
338  std::cout << 100.0*static_cast<double>(statVector[icol][irow])/static_cast<double>(dataVector[icol].size());
339  else
340  std::cout << statVector[icol][irow];
341  if(icol<col_opt.size()-1)
342  cout << " ";
343  }
344  cout << endl;
345  }
346  }
347  if(histogram2d_opt[0]){
348  assert(nbin);
349  assert(dataVector.size()==2);
350  assert(dataVector[0].size()==dataVector[1].size());
351  double sigma=0;
352  //kernel density estimation as in http://en.wikipedia.org/wiki/Kernel_density_estimation
353  if(kde_opt[0]){
354  // if(kde_opt[0]>0)
355  // sigma=kde_opt[0];
356  // else
357  sigma=1.06*sqrt(sqrt(stat.var(dataVector[0]))*sqrt(stat.var(dataVector[1])))*pow(dataVector[0].size(),-0.2);
358  }
359  assert(nbin);
360  if(verbose_opt[0]){
361  if(sigma>0)
362  std::cout << "calculating 2d kernel density estimate with sigma " << sigma << " for cols " << col_opt[0] << " and " << col_opt[1] << std::endl;
363  else
364  std::cout << "calculating 2d histogram for cols " << col_opt[0] << " and " << col_opt[1] << std::endl;
365  std::cout << "nbin: " << nbin << std::endl;
366  }
367  std::vector< std::vector<double> > histVector;
368  stat.distribution2d(dataVector[0],dataVector[1],histVector,nbin,minX,maxX,minY,maxY,sigma);
369  for(int binX=0;binX<nbin;++binX){
370  std::cout << std::endl;
371  for(int binY=0;binY<nbin;++binY){
372  double binValueX=0;
373  if(nbin==maxX-minX+1)
374  binValueX=minX+binX;
375  else
376  binValueX=minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
377  double binValueY=0;
378  if(nbin==maxY-minY+1)
379  binValueY=minY+binY;
380  else
381  binValueY=minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
382  double value=0;
383  value=static_cast<double>(histVector[binX][binY])/dataVector[0].size();
384  std::cout << binValueX << " " << binValueY << " " << value << std::endl;
385  // std::cout << minX+static_cast<double>(maxX-minX)*(binX+0.5)/nbin << " " << minY+static_cast<double>(maxY-minY)*(binY+0.5)/nbin << " " << value << std::endl;
386  }
387  }
388  }
389 
390  if(output_opt[0]){
391  if(transpose_opt[0]){
392  for(int icol=0;icol<col_opt.size();++icol){
393  for(int irow=0;irow<dataVector.begin()->size();++irow){
394  cout << dataVector[icol][irow];
395  if(irow<dataVector.begin()->size()-1)
396  cout << " ";
397  }
398  cout << endl;
399  }
400  }
401  else{
402  for(int irow=0;irow<dataVector.begin()->size();++irow){
403  for(int icol=0;icol<col_opt.size();++icol){
404  cout << dataVector[icol][irow];
405  if(icol<col_opt.size()-1)
406  cout << " ";
407  }
408  cout << endl;
409  }
410  }
411  }
412 }