24 #include "base/Optionpk.h"
25 #include "fileclasses/FileReaderAscii.h"
26 #include "algorithms/StatFactory.h"
29 int main(
int argc,
char *argv[])
33 Optionpk<char> comment_opt(
"comment",
"comment",
"comment character",
'#');
34 Optionpk<bool> output_opt(
"o",
"output",
"output the selected columns",
false);
35 Optionpk<bool> transpose_opt(
"t",
"transpose",
"transpose input ascii vector (use in combination with --output)",
false);
36 Optionpk<int> col_opt(
"c",
"column",
"column nr, starting from 0", 0);
37 Optionpk<int> range_opt(
"r",
"range",
"rows to start/end reading. Use -r 1 -r 10 to read first 10 rows where first row is header. Use 0 to read all rows with no header.", 0);
40 Optionpk<std::string> randdist_opt(
"dist",
"dist",
"distribution for generating random numbers, see http://www.gn/software/gsl/manual/gsl-ref_toc.html#TOC320 (only uniform and Gaussian supported yet)",
"gaussian");
41 Optionpk<double> randa_opt(
"rnda",
"rnda",
"first parameter for random distribution (mean value in case of Gaussian)", 0);
42 Optionpk<double> randb_opt(
"rndb",
"rndb",
"second parameter for random distribution (standard deviation in case of Gaussian)", 1);
44 Optionpk<bool> median_opt(
"median",
"median",
"calculate median",
false);
46 Optionpk<bool> skewness_opt(
"skew",
"skewness",
"calculate skewness",
false);
47 Optionpk<bool> kurtosis_opt(
"kurt",
"kurtosis",
"calculate kurtosis",
false);
48 Optionpk<bool> stdev_opt(
"stdev",
"stdev",
"calculate standard deviation",
false);
49 Optionpk<bool> sum_opt(
"sum",
"sum",
"calculate sum of column",
false);
50 Optionpk<bool> minmax_opt(
"mm",
"minmax",
"calculate minimum and maximum value",
false);
51 Optionpk<bool> min_opt(
"min",
"min",
"calculate minimum value",
false);
52 Optionpk<bool> max_opt(
"max",
"max",
"calculate maximum value",
false);
53 Optionpk<double> src_min_opt(
"src_min",
"src_min",
"start reading source from this minimum value");
54 Optionpk<double> src_max_opt(
"src_max",
"src_max",
"stop reading source from this maximum value");
55 Optionpk<bool> histogram_opt(
"hist",
"hist",
"calculate histogram",
false);
56 Optionpk<bool> histogram2d_opt(
"hist2d",
"hist2d",
"calculate 2-dimensional histogram based on two columns",
false);
57 Optionpk<short> nbin_opt(
"nbin",
"nbin",
"number of bins to calculate histogram");
58 Optionpk<bool> relative_opt(
"rel",
"relative",
"use percentiles for histogram to calculate histogram",
false);
59 Optionpk<bool> kde_opt(
"kde",
"kde",
"Use Kernel density estimation when producing histogram. The standard deviation is estimated based on Silverman's rule of thumb",
false);
60 Optionpk<bool> correlation_opt(
"cor",
"correlation",
"calculate Pearson produc-moment correlation coefficient between two columns (defined by -c <col1> -c <col2>",
false);
61 Optionpk<bool> rmse_opt(
"rmse",
"rmse",
"calculate root mean square error between two columns (defined by -c <col1> -c <col2>",
false);
62 Optionpk<bool> reg_opt(
"reg",
"regression",
"calculate linear regression between two columns and get correlation coefficient (defined by -c <col1> -c <col2>",
false);
63 Optionpk<bool> regerr_opt(
"regerr",
"regerr",
"calculate linear regression between two columns and get root mean square error (defined by -c <col1> -c <col2>",
false);
64 Optionpk<short> verbose_opt(
"v",
"verbose",
"verbose mode when positive", 0,2);
66 src_min_opt.setHide(1);
67 src_max_opt.setHide(1);
70 output_opt.setHide(1);
71 transpose_opt.setHide(1);
72 comment_opt.setHide(1);
77 doProcess=input_opt.retrieveOption(argc,argv);
78 col_opt.retrieveOption(argc,argv);
80 size_opt.retrieveOption(argc,argv);
81 rand_opt.retrieveOption(argc,argv);
82 randdist_opt.retrieveOption(argc,argv);
83 randa_opt.retrieveOption(argc,argv);
84 randb_opt.retrieveOption(argc,argv);
85 mean_opt.retrieveOption(argc,argv);
86 median_opt.retrieveOption(argc,argv);
87 var_opt.retrieveOption(argc,argv);
88 stdev_opt.retrieveOption(argc,argv);
89 skewness_opt.retrieveOption(argc,argv);
90 kurtosis_opt.retrieveOption(argc,argv);
91 sum_opt.retrieveOption(argc,argv);
92 minmax_opt.retrieveOption(argc,argv);
93 min_opt.retrieveOption(argc,argv);
94 max_opt.retrieveOption(argc,argv);
95 histogram_opt.retrieveOption(argc,argv);
96 nbin_opt.retrieveOption(argc,argv);
97 relative_opt.retrieveOption(argc,argv);
98 kde_opt.retrieveOption(argc,argv);
99 histogram2d_opt.retrieveOption(argc,argv);
100 correlation_opt.retrieveOption(argc,argv);
101 rmse_opt.retrieveOption(argc,argv);
102 reg_opt.retrieveOption(argc,argv);
103 regerr_opt.retrieveOption(argc,argv);
105 src_min_opt.retrieveOption(argc,argv);
106 src_max_opt.retrieveOption(argc,argv);
107 fs_opt.retrieveOption(argc,argv);
108 range_opt.retrieveOption(argc,argv);
109 output_opt.retrieveOption(argc,argv);
110 transpose_opt.retrieveOption(argc,argv);
111 comment_opt.retrieveOption(argc,argv);
112 verbose_opt.retrieveOption(argc,argv);
114 catch(
string predefinedString){
115 std::cout << predefinedString << std::endl;
120 cout <<
"Usage: pkstatascii -i input [-c column]*" << endl;
122 std::cout <<
"short option -h shows basic options only, use long option --help to show all options" << std::endl;
126 if(src_min_opt.size()){
127 while(src_min_opt.size()<col_opt.size())
128 src_min_opt.push_back(src_min_opt[0]);
130 if(src_max_opt.size()){
131 while(src_max_opt.size()<col_opt.size())
132 src_max_opt.push_back(src_max_opt[0]);
136 gsl_rng* r=stat.getRandomGenerator(time(NULL));
139 std::cout <<
"generating " << rand_opt[0] <<
" random numbers: " << std::endl;
140 for(
unsigned int i=0;i<rand_opt[0];++i)
141 std::cout << i <<
" " << stat.getRandomValue(r,randdist_opt[0],randa_opt[0],randb_opt[0]) << std::endl;
143 vector< vector<double> > dataVector(col_opt.size());
144 vector< vector<double> > statVector(col_opt.size());
146 if(!input_opt.size())
149 asciiReader.setFieldSeparator(fs_opt[0]);
150 asciiReader.setComment(comment_opt[0]);
151 asciiReader.setMinRow(range_opt[0]);
152 if(range_opt.size()>1)
153 asciiReader.setMaxRow(range_opt[1]);
154 asciiReader.readData(dataVector,col_opt);
155 assert(dataVector.size());
161 if(histogram_opt[0]){
162 stat.minmax(dataVector[0],dataVector[0].begin(),dataVector[0].end(),minValue,maxValue);
163 if(src_min_opt.size())
164 minValue=src_min_opt[0];
165 if(src_max_opt.size())
166 maxValue=src_max_opt[0];
168 std::cerr <<
"Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
169 nbin=maxValue-minValue+1;
176 if(histogram2d_opt[0]){
177 assert(col_opt.size()==2);
179 std::cerr <<
"Warning: number of bins not defined, calculating bins from min and max value" << std::endl;
180 stat.minmax(dataVector[0],dataVector[0].begin(),dataVector[0].end(),minX,maxX);
181 stat.minmax(dataVector[1],dataVector[1].begin(),dataVector[1].end(),minY,maxY);
182 if(src_min_opt.size())
184 if(src_min_opt.size()>1)
186 if(src_max_opt.size())
188 if(src_max_opt.size()>1)
190 minValue=(minX<minY)? minX:minY;
191 maxValue=(maxX>maxY)? maxX:maxY;
193 std::cout <<
"min and max values: " << minValue <<
", " << maxValue << std::endl;
194 nbin=maxValue-minValue+1;
197 for(
int icol=0;icol<col_opt.size();++icol){
198 if(!dataVector[icol].size()){
199 std::cerr <<
"Warning: dataVector[" << icol <<
"] is empty" << std::endl;
203 cout <<
"sample size column " << col_opt[icol] <<
": " << dataVector[icol].size() << endl;
205 cout <<
"mean value column " << col_opt[icol] <<
": " << stat.mean(dataVector[icol]) << endl;
207 cout <<
"variance value column " << col_opt[icol] <<
": " << stat.var(dataVector[icol]) << endl;
209 cout <<
"standard deviation column " << col_opt[icol] <<
": " << sqrt(stat.var(dataVector[icol])) << endl;
211 cout <<
"skewness value column " << col_opt[icol] <<
": " << stat.skewness(dataVector[icol]) << endl;
213 cout <<
"kurtosis value column " << col_opt[icol] <<
": " << stat.kurtosis(dataVector[icol]) << endl;
215 cout << setprecision(2);
216 cout << fixed <<
"sum column " << col_opt[icol] <<
": " << (stat.sum(dataVector[icol])) << endl;
219 cout <<
"median value column " << col_opt[icol] <<
": " << stat.median(dataVector[icol]) << endl;
221 cout <<
"min value column " << col_opt[icol] <<
": " << stat.mymin(dataVector[icol]) << endl;
222 cout <<
"max value column " << col_opt[icol] <<
": " << stat.mymax(dataVector[icol]) << endl;
225 cout <<
"min value column " << col_opt[icol] <<
": " << stat.mymin(dataVector[icol]) << endl;
227 cout <<
"max value column " << col_opt[icol] <<
": " << stat.mymax(dataVector[icol]) << endl;
228 if(histogram_opt[0]){
235 sigma=1.06*sqrt(stat.var(dataVector[icol]))*pow(dataVector[icol].size(),-0.2);
240 std::cout <<
"calculating kernel density estimate with sigma " << sigma <<
" for col " << icol << std::endl;
242 std::cout <<
"calculating histogram for col " << icol << std::endl;
297 stat.distribution(dataVector[icol],dataVector[icol].begin(),dataVector[icol].end(),statVector[icol],nbin,minValue,maxValue,sigma);
299 std::cout <<
"min and max values: " << minValue <<
", " << maxValue << std::endl;
302 if(correlation_opt[0]){
303 assert(dataVector.size()==2);
304 cout <<
"correlation between columns " << col_opt[0] <<
" and " << col_opt[1] <<
": " << stat.correlation(dataVector[0],dataVector[1]) << endl;
307 assert(dataVector.size()==2);
308 cout <<
"root mean square error between columns " << col_opt[0] <<
" and " << col_opt[1] <<
": " << stat.rmse(dataVector[0],dataVector[1]) << endl;
311 assert(dataVector.size()==2);
314 double r2=stat.linear_regression(dataVector[0],dataVector[1],c0,c1);
315 cout <<
"linear regression between columns: " << col_opt[0] <<
" and " << col_opt[1] <<
": " << c0 <<
"+" << c1 <<
" * x " <<
" with R^2 (square correlation coefficient): " << r2 << endl;
318 assert(dataVector.size()==2);
321 double err=stat.linear_regression_err(dataVector[0],dataVector[1],c0,c1);
323 cout <<
"linear regression between columns: " << col_opt[0] <<
" and " << col_opt[1] <<
": " << c0 <<
"+" << c1 <<
" * x " <<
" with rmse: " << err << endl;
325 cout << c0 <<
" " << c1 <<
" " << err << endl;
327 if(histogram_opt[0]){
328 for(
int irow=0;irow<statVector.begin()->size();++irow){
330 if(nbin==maxValue-minValue+1)
331 binValue=minValue+irow;
333 binValue=minValue+
static_cast<double>(maxValue-minValue)*(irow+0.5)/nbin;
334 std::cout << binValue <<
" ";
336 for(
int icol=0;icol<col_opt.size();++icol){
338 std::cout << 100.0*
static_cast<double>(statVector[icol][irow])/static_cast<double>(dataVector[icol].size());
340 std::cout << statVector[icol][irow];
341 if(icol<col_opt.size()-1)
347 if(histogram2d_opt[0]){
349 assert(dataVector.size()==2);
350 assert(dataVector[0].size()==dataVector[1].size());
357 sigma=1.06*sqrt(sqrt(stat.var(dataVector[0]))*sqrt(stat.var(dataVector[1])))*pow(dataVector[0].size(),-0.2);
362 std::cout <<
"calculating 2d kernel density estimate with sigma " << sigma <<
" for cols " << col_opt[0] <<
" and " << col_opt[1] << std::endl;
364 std::cout <<
"calculating 2d histogram for cols " << col_opt[0] <<
" and " << col_opt[1] << std::endl;
365 std::cout <<
"nbin: " << nbin << std::endl;
367 std::vector< std::vector<double> > histVector;
368 stat.distribution2d(dataVector[0],dataVector[1],histVector,nbin,minX,maxX,minY,maxY,sigma);
369 for(
int binX=0;binX<nbin;++binX){
370 std::cout << std::endl;
371 for(
int binY=0;binY<nbin;++binY){
373 if(nbin==maxX-minX+1)
376 binValueX=minX+
static_cast<double>(maxX-minX)*(binX+0.5)/nbin;
378 if(nbin==maxY-minY+1)
381 binValueY=minY+
static_cast<double>(maxY-minY)*(binY+0.5)/nbin;
383 value=
static_cast<double>(histVector[binX][binY])/dataVector[0].size();
384 std::cout << binValueX <<
" " << binValueY <<
" " << value << std::endl;
391 if(transpose_opt[0]){
392 for(
int icol=0;icol<col_opt.size();++icol){
393 for(
int irow=0;irow<dataVector.begin()->size();++irow){
394 cout << dataVector[icol][irow];
395 if(irow<dataVector.begin()->size()-1)
402 for(
int irow=0;irow<dataVector.begin()->size();++irow){
403 for(
int icol=0;icol<col_opt.size();++icol){
404 cout << dataVector[icol][irow];
405 if(icol<col_opt.size()-1)