1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package org.hipparchus.stat.inference;
23
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.util.ArrayList;
30 import java.util.List;
31
32 import org.hipparchus.distribution.continuous.NormalDistribution;
33 import org.hipparchus.exception.MathIllegalArgumentException;
34 import org.hipparchus.exception.NullArgumentException;
35 import org.hipparchus.stat.descriptive.StreamingStatistics;
36 import org.hipparchus.util.FastMath;
37 import org.junit.Test;
38
39
40
41
42
43 public class InferenceTestUtilsTest {
44
45 private double[] classA = { 93.0, 103.0, 95.0, 101.0 };
46 private double[] classB = { 99.0, 92.0, 102.0, 100.0, 102.0 };
47 private double[] classC = { 110.0, 115.0, 111.0, 117.0, 128.0 };
48
49 private List<double[]> classes = new ArrayList<double[]>();
50 private OneWayAnova oneWayAnova = new OneWayAnova();
51
52
53 @Test
54 public void testChiSquare() {
55
56
57
58
59
60
61 long[] observed = {10, 9, 11};
62 double[] expected = {10, 10, 10};
63 assertEquals("chi-square statistic", 0.2,
64 InferenceTestUtils.chiSquare(expected, observed), 10E-12);
65 assertEquals("chi-square p-value", 0.904837418036,
66 InferenceTestUtils.chiSquareTest(expected, observed), 1E-10);
67
68 long[] observed1 = { 500, 623, 72, 70, 31 };
69 double[] expected1 = { 485, 541, 82, 61, 37 };
70 assertEquals("chi-square test statistic", 9.023307936427388,
71 InferenceTestUtils.chiSquare(expected1, observed1), 1E-10);
72 assertEquals("chi-square p-value", 0.06051952647453607,
73 InferenceTestUtils.chiSquareTest(expected1, observed1), 1E-9);
74 assertTrue("chi-square test reject",
75 InferenceTestUtils.chiSquareTest(expected1, observed1, 0.07));
76 assertTrue("chi-square test accept",
77 !InferenceTestUtils.chiSquareTest(expected1, observed1, 0.05));
78
79 try {
80 InferenceTestUtils.chiSquareTest(expected1, observed1, 95);
81 fail("alpha out of range, MathIllegalArgumentException expected");
82 } catch (MathIllegalArgumentException ex) {
83
84 }
85
86 long[] tooShortObs = { 0 };
87 double[] tooShortEx = { 1 };
88 try {
89 InferenceTestUtils.chiSquare(tooShortEx, tooShortObs);
90 fail("arguments too short, MathIllegalArgumentException expected");
91 } catch (MathIllegalArgumentException ex) {
92
93 }
94
95
96 long[] unMatchedObs = { 0, 1, 2, 3 };
97 double[] unMatchedEx = { 1, 1, 2 };
98 try {
99 InferenceTestUtils.chiSquare(unMatchedEx, unMatchedObs);
100 fail("arrays have different lengths, MathIllegalArgumentException expected");
101 } catch (MathIllegalArgumentException ex) {
102
103 }
104
105
106 expected[0] = 0;
107 try {
108 InferenceTestUtils.chiSquareTest(expected, observed, .01);
109 fail("bad expected count, MathIllegalArgumentException expected");
110 } catch (MathIllegalArgumentException ex) {
111
112 }
113
114
115 expected[0] = 1;
116 observed[0] = -1;
117 try {
118 InferenceTestUtils.chiSquareTest(expected, observed, .01);
119 fail("bad expected count, MathIllegalArgumentException expected");
120 } catch (MathIllegalArgumentException ex) {
121
122 }
123
124 }
125
126 @Test
127 public void testChiSquareIndependence() {
128
129
130
131 long[][] counts = { {40, 22, 43}, {91, 21, 28}, {60, 10, 22}};
132 assertEquals( "chi-square test statistic", 22.709027688, InferenceTestUtils.chiSquare(counts), 1E-9);
133 assertEquals("chi-square p-value", 0.000144751460134, InferenceTestUtils.chiSquareTest(counts), 1E-9);
134 assertTrue("chi-square test reject", InferenceTestUtils.chiSquareTest(counts, 0.0002));
135 assertTrue("chi-square test accept", !InferenceTestUtils.chiSquareTest(counts, 0.0001));
136
137 long[][] counts2 = {{10, 15}, {30, 40}, {60, 90} };
138 assertEquals( "chi-square test statistic", 0.168965517241, InferenceTestUtils.chiSquare(counts2), 1E-9);
139 assertEquals("chi-square p-value",0.918987499852, InferenceTestUtils.chiSquareTest(counts2), 1E-9);
140 assertTrue("chi-square test accept", !InferenceTestUtils.chiSquareTest(counts2, 0.1));
141
142
143 long[][] counts3 = { {40, 22, 43}, {91, 21, 28}, {60, 10}};
144 try {
145 InferenceTestUtils.chiSquare(counts3);
146 fail("Expecting MathIllegalArgumentException");
147 } catch (MathIllegalArgumentException ex) {
148
149 }
150
151
152 long[][] counts4 = {{40, 22, 43}};
153 try {
154 InferenceTestUtils.chiSquare(counts4);
155 fail("Expecting MathIllegalArgumentException");
156 } catch (MathIllegalArgumentException ex) {
157
158 }
159 long[][] counts5 = {{40}, {40}, {30}, {10}};
160 try {
161 InferenceTestUtils.chiSquare(counts5);
162 fail("Expecting MathIllegalArgumentException");
163 } catch (MathIllegalArgumentException ex) {
164
165 }
166
167
168 long[][] counts6 = {{10, -2}, {30, 40}, {60, 90} };
169 try {
170 InferenceTestUtils.chiSquare(counts6);
171 fail("Expecting MathIllegalArgumentException");
172 } catch (MathIllegalArgumentException ex) {
173
174 }
175
176
177 try {
178 InferenceTestUtils.chiSquareTest(counts, 0);
179 fail("Expecting MathIllegalArgumentException");
180 } catch (MathIllegalArgumentException ex) {
181
182 }
183 }
184
185 @Test
186 public void testChiSquareLargeTestStatistic() {
187 double[] exp = new double[] {
188 3389119.5, 649136.6, 285745.4, 25357364.76,
189 11291189.78, 543628.0, 232921.0, 437665.75
190 };
191
192 long[] obs = new long[] { 2372383, 584222, 257170, 17750155, 7903832, 489265, 209628, 393899 };
193
194 ChiSquareTest csti = new ChiSquareTest();
195 double cst = csti.chiSquareTest(exp, obs);
196 assertEquals("chi-square p-value", 0.0, cst, 1E-3);
197 assertEquals("chi-square test statistic", 114875.90421929007,
198 InferenceTestUtils.chiSquare(exp, obs), 1E-9);
199 }
200
201
202 @Test
203 public void testChiSquareZeroCount() {
204
205 long[][] counts = { {40, 0, 4}, {91, 1, 2}, {60, 2, 0}};
206 assertEquals( "chi-square test statistic", 9.67444662263, InferenceTestUtils.chiSquare(counts), 1E-9);
207 assertEquals("chi-square p-value", 0.0462835770603, InferenceTestUtils.chiSquareTest(counts), 1E-9);
208 }
209
210 private double[] tooShortObs = { 1.0 };
211 private double[] emptyObs = {};
212 private StreamingStatistics emptyStats = new StreamingStatistics();
213
214 @Test
215 public void testOneSampleT() {
216 double[] observed = {
217 93.0, 103.0, 95.0, 101.0, 91.0, 105.0, 96.0, 94.0,
218 101.0, 88.0, 98.0, 94.0, 101.0, 92.0, 95.0
219 };
220 double mu = 100.0;
221 StreamingStatistics sampleStats = new StreamingStatistics();
222 for (int i = 0; i < observed.length; i++) {
223 sampleStats.addValue(observed[i]);
224 }
225
226
227 assertEquals("t statistic", -2.81976445346, InferenceTestUtils.t(mu, observed), 10E-10);
228 assertEquals("t statistic", -2.81976445346, InferenceTestUtils.t(mu, sampleStats), 10E-10);
229 assertEquals("p value", 0.0136390585873, InferenceTestUtils.tTest(mu, observed), 10E-10);
230 assertEquals("p value", 0.0136390585873, InferenceTestUtils.tTest(mu, sampleStats), 10E-10);
231
232 try {
233 InferenceTestUtils.t(mu, (double[]) null);
234 fail("arguments too short, NullArgumentException expected");
235 } catch (NullArgumentException ex) {
236
237 }
238
239 try {
240 InferenceTestUtils.t(mu, (StreamingStatistics) null);
241 fail("arguments too short, NullArgumentException expected");
242 } catch (NullArgumentException ex) {
243
244 }
245
246 try {
247 InferenceTestUtils.t(mu, emptyObs);
248 fail("arguments too short, MathIllegalArgumentException expected");
249 } catch (MathIllegalArgumentException ex) {
250
251 }
252
253 try {
254 InferenceTestUtils.t(mu, emptyStats);
255 fail("arguments too short, MathIllegalArgumentException expected");
256 } catch (MathIllegalArgumentException ex) {
257
258 }
259
260 try {
261 InferenceTestUtils.t(mu, tooShortObs);
262 fail("insufficient data to compute t statistic, MathIllegalArgumentException expected");
263 } catch (MathIllegalArgumentException ex) {
264
265 }
266 try {
267 InferenceTestUtils.tTest(mu, tooShortObs);
268 fail("insufficient data to perform t test, MathIllegalArgumentException expected");
269 } catch (MathIllegalArgumentException ex) {
270
271 }
272
273 try {
274 InferenceTestUtils.t(mu, (StreamingStatistics) null);
275 fail("insufficient data to compute t statistic, NullArgumentException expected");
276 } catch (NullArgumentException ex) {
277
278 }
279 try {
280 InferenceTestUtils.tTest(mu, (StreamingStatistics) null);
281 fail("insufficient data to perform t test, NullArgumentException expected");
282 } catch (NullArgumentException ex) {
283
284 }
285 }
286
287 @Test
288 public void testOneSampleTTest() {
289 double[] oneSidedP = {
290 2d, 0d, 6d, 6d, 3d, 3d, 2d, 3d, -6d, 6d,
291 6d, 6d, 3d, 0d, 1d, 1d, 0d, 2d, 3d, 3d
292 };
293 StreamingStatistics oneSidedPStats = new StreamingStatistics();
294 for (int i = 0; i < oneSidedP.length; i++) {
295 oneSidedPStats.addValue(oneSidedP[i]);
296 }
297
298 assertEquals("one sample t stat", 3.86485535541, InferenceTestUtils.t(0d, oneSidedP), 10E-10);
299 assertEquals("one sample t stat", 3.86485535541, InferenceTestUtils.t(0d, oneSidedPStats),1E-10);
300 assertEquals("one sample p value", 0.000521637019637, InferenceTestUtils.tTest(0d, oneSidedP) / 2d, 10E-10);
301 assertEquals("one sample p value", 0.000521637019637, InferenceTestUtils.tTest(0d, oneSidedPStats) / 2d, 10E-5);
302 assertTrue("one sample t-test reject", InferenceTestUtils.tTest(0d, oneSidedP, 0.01));
303 assertTrue("one sample t-test reject", InferenceTestUtils.tTest(0d, oneSidedPStats, 0.01));
304 assertTrue("one sample t-test accept", !InferenceTestUtils.tTest(0d, oneSidedP, 0.0001));
305 assertTrue("one sample t-test accept", !InferenceTestUtils.tTest(0d, oneSidedPStats, 0.0001));
306
307 try {
308 InferenceTestUtils.tTest(0d, oneSidedP, 95);
309 fail("alpha out of range, MathIllegalArgumentException expected");
310 } catch (MathIllegalArgumentException ex) {
311
312 }
313
314 try {
315 InferenceTestUtils.tTest(0d, oneSidedPStats, 95);
316 fail("alpha out of range, MathIllegalArgumentException expected");
317 } catch (MathIllegalArgumentException ex) {
318
319 }
320
321 }
322
323 @Test
324 public void testTwoSampleTHeterscedastic() {
325 double[] sample1 = { 7d, -4d, 18d, 17d, -3d, -5d, 1d, 10d, 11d, -2d };
326 double[] sample2 = { -1d, 12d, -1d, -3d, 3d, -5d, 5d, 2d, -11d, -1d, -3d };
327 StreamingStatistics sampleStats1 = new StreamingStatistics();
328 for (int i = 0; i < sample1.length; i++) {
329 sampleStats1.addValue(sample1[i]);
330 }
331 StreamingStatistics sampleStats2 = new StreamingStatistics();
332 for (int i = 0; i < sample2.length; i++) {
333 sampleStats2.addValue(sample2[i]);
334 }
335
336
337 assertEquals("two sample heteroscedastic t stat", 1.60371728768,
338 InferenceTestUtils.t(sample1, sample2), 1E-10);
339 assertEquals("two sample heteroscedastic t stat", 1.60371728768,
340 InferenceTestUtils.t(sampleStats1, sampleStats2), 1E-10);
341 assertEquals("two sample heteroscedastic p value", 0.128839369622,
342 InferenceTestUtils.tTest(sample1, sample2), 1E-10);
343 assertEquals("two sample heteroscedastic p value", 0.128839369622,
344 InferenceTestUtils.tTest(sampleStats1, sampleStats2), 1E-10);
345 assertTrue("two sample heteroscedastic t-test reject",
346 InferenceTestUtils.tTest(sample1, sample2, 0.2));
347 assertTrue("two sample heteroscedastic t-test reject",
348 InferenceTestUtils.tTest(sampleStats1, sampleStats2, 0.2));
349 assertTrue("two sample heteroscedastic t-test accept",
350 !InferenceTestUtils.tTest(sample1, sample2, 0.1));
351 assertTrue("two sample heteroscedastic t-test accept",
352 !InferenceTestUtils.tTest(sampleStats1, sampleStats2, 0.1));
353
354 try {
355 InferenceTestUtils.tTest(sample1, sample2, .95);
356 fail("alpha out of range, MathIllegalArgumentException expected");
357 } catch (MathIllegalArgumentException ex) {
358
359 }
360
361 try {
362 InferenceTestUtils.tTest(sampleStats1, sampleStats2, .95);
363 fail("alpha out of range, MathIllegalArgumentException expected");
364 } catch (MathIllegalArgumentException ex) {
365
366 }
367
368 try {
369 InferenceTestUtils.tTest(sample1, tooShortObs, .01);
370 fail("insufficient data, MathIllegalArgumentException expected");
371 } catch (MathIllegalArgumentException ex) {
372
373 }
374
375 try {
376 InferenceTestUtils.tTest(sampleStats1, (StreamingStatistics) null, .01);
377 fail("insufficient data, NullArgumentException expected");
378 } catch (NullArgumentException ex) {
379
380 }
381
382 try {
383 InferenceTestUtils.tTest(sample1, tooShortObs);
384 fail("insufficient data, MathIllegalArgumentException expected");
385 } catch (MathIllegalArgumentException ex) {
386
387 }
388
389 try {
390 InferenceTestUtils.tTest(sampleStats1, (StreamingStatistics) null);
391 fail("insufficient data, NullArgumentException expected");
392 } catch (NullArgumentException ex) {
393
394 }
395
396 try {
397 InferenceTestUtils.t(sample1, tooShortObs);
398 fail("insufficient data, MathIllegalArgumentException expected");
399 } catch (MathIllegalArgumentException ex) {
400
401 }
402
403 try {
404 InferenceTestUtils.t(sampleStats1, (StreamingStatistics) null);
405 fail("insufficient data, NullArgumentException expected");
406 } catch (NullArgumentException ex) {
407
408 }
409 }
410 @Test
411 public void testTwoSampleTHomoscedastic() {
412 double[] sample1 ={2, 4, 6, 8, 10, 97};
413 double[] sample2 = {4, 6, 8, 10, 16};
414 StreamingStatistics sampleStats1 = new StreamingStatistics();
415 for (int i = 0; i < sample1.length; i++) {
416 sampleStats1.addValue(sample1[i]);
417 }
418 StreamingStatistics sampleStats2 = new StreamingStatistics();
419 for (int i = 0; i < sample2.length; i++) {
420 sampleStats2.addValue(sample2[i]);
421 }
422
423
424 assertEquals("two sample homoscedastic t stat", 0.73096310086,
425 InferenceTestUtils.homoscedasticT(sample1, sample2), 10E-11);
426 assertEquals("two sample homoscedastic p value", 0.4833963785,
427 InferenceTestUtils.homoscedasticTTest(sampleStats1, sampleStats2), 1E-10);
428 assertTrue("two sample homoscedastic t-test reject",
429 InferenceTestUtils.homoscedasticTTest(sample1, sample2, 0.49));
430 assertTrue("two sample homoscedastic t-test accept",
431 !InferenceTestUtils.homoscedasticTTest(sample1, sample2, 0.48));
432 }
433
434 @Test
435 public void testSmallSamples() {
436 double[] sample1 = {1d, 3d};
437 double[] sample2 = {4d, 5d};
438
439
440 assertEquals(-2.2360679775, InferenceTestUtils.t(sample1, sample2), 1E-10);
441 assertEquals(0.198727388935, InferenceTestUtils.tTest(sample1, sample2), 1E-10);
442 }
443
444 @Test
445 public void testPaired() {
446 double[] sample1 = {1d, 3d, 5d, 7d};
447 double[] sample2 = {0d, 6d, 11d, 2d};
448 double[] sample3 = {5d, 7d, 8d, 10d};
449
450
451 assertEquals(-0.3133, InferenceTestUtils.pairedT(sample1, sample2), 1E-4);
452 assertEquals(0.774544295819, InferenceTestUtils.pairedTTest(sample1, sample2), 1E-10);
453 assertEquals(0.001208, InferenceTestUtils.pairedTTest(sample1, sample3), 1E-6);
454 assertFalse(InferenceTestUtils.pairedTTest(sample1, sample3, .001));
455 assertTrue(InferenceTestUtils.pairedTTest(sample1, sample3, .002));
456 }
457
458 @Test
459 public void testOneWayAnovaUtils() {
460 classes.add(classA);
461 classes.add(classB);
462 classes.add(classC);
463 assertEquals(oneWayAnova.anovaFValue(classes),
464 InferenceTestUtils.oneWayAnovaFValue(classes), 10E-12);
465 assertEquals(oneWayAnova.anovaPValue(classes),
466 InferenceTestUtils.oneWayAnovaPValue(classes), 10E-12);
467 assertEquals(oneWayAnova.anovaTest(classes, 0.01),
468 InferenceTestUtils.oneWayAnovaTest(classes, 0.01));
469 }
470 @Test
471 public void testGTestGoodnesOfFit() throws Exception {
472 double[] exp = new double[] { 0.54d, 0.40d, 0.05d, 0.01d };
473 long[] obs = new long[] { 70, 79, 3, 4 };
474
475 assertEquals("G test statistic", 13.144799, InferenceTestUtils.g(exp, obs), 1E-5);
476 double p_gtgf = InferenceTestUtils.gTest(exp, obs);
477 assertEquals("g-Test p-value", 0.004333, p_gtgf, 1E-5);
478 assertTrue(InferenceTestUtils.gTest(exp, obs, 0.05));
479 }
480
481 @Test
482 public void testGTestIndependance() throws Exception {
483 long[] obs1 = new long[] { 268, 199, 42 };
484 long[] obs2 = new long[] { 807, 759, 184 };
485
486 double g = InferenceTestUtils.gDataSetsComparison(obs1, obs2);
487
488 assertEquals("G test statistic", 7.3008170, g, 1E-4);
489 double p_gti = InferenceTestUtils.gTestDataSetsComparison(obs1, obs2);
490
491 assertEquals("g-Test p-value", 0.0259805, p_gti, 1E-4);
492 assertTrue(InferenceTestUtils.gTestDataSetsComparison(obs1, obs2, 0.05));
493 }
494
495 @Test
496 public void testRootLogLikelihood() {
497
498 assertTrue(InferenceTestUtils.rootLogLikelihoodRatio(904, 21060, 1144, 283012) > 0.0);
499
500
501 assertTrue(InferenceTestUtils.rootLogLikelihoodRatio(36, 21928, 60280, 623876) < 0.0);
502
503 assertEquals(FastMath.sqrt(2.772589), InferenceTestUtils.rootLogLikelihoodRatio(1, 0, 0, 1), 0.000001);
504 assertEquals(-FastMath.sqrt(2.772589), InferenceTestUtils.rootLogLikelihoodRatio(0, 1, 1, 0), 0.000001);
505 assertEquals(FastMath.sqrt(27.72589), InferenceTestUtils.rootLogLikelihoodRatio(10, 0, 0, 10), 0.00001);
506
507 assertEquals(FastMath.sqrt(39.33052), InferenceTestUtils.rootLogLikelihoodRatio(5, 1995, 0, 100000), 0.00001);
508 assertEquals(-FastMath.sqrt(39.33052), InferenceTestUtils.rootLogLikelihoodRatio(0, 100000, 5, 1995), 0.00001);
509
510 assertEquals(FastMath.sqrt(4730.737), InferenceTestUtils.rootLogLikelihoodRatio(1000, 1995, 1000, 100000), 0.001);
511 assertEquals(-FastMath.sqrt(4730.737), InferenceTestUtils.rootLogLikelihoodRatio(1000, 100000, 1000, 1995), 0.001);
512
513 assertEquals(FastMath.sqrt(5734.343), InferenceTestUtils.rootLogLikelihoodRatio(1000, 1000, 1000, 100000), 0.001);
514 assertEquals(FastMath.sqrt(5714.932), InferenceTestUtils.rootLogLikelihoodRatio(1000, 1000, 1000, 99000), 0.001);
515 }
516
517 @Test
518 public void testKSOneSample() throws Exception {
519 final NormalDistribution unitNormal = new NormalDistribution(0d, 1d);
520 final double[] sample = KolmogorovSmirnovTestTest.gaussian;
521 final double tol = KolmogorovSmirnovTestTest.TOLERANCE;
522 assertEquals(0.3172069207622391, InferenceTestUtils.kolmogorovSmirnovTest(unitNormal, sample), tol);
523 assertEquals(0.0932947561266756, InferenceTestUtils.kolmogorovSmirnovStatistic(unitNormal, sample), tol);
524 }
525
526 @Test
527 public void testKSTwoSample() throws Exception {
528 final double tol = KolmogorovSmirnovTestTest.TOLERANCE;
529 final double[] smallSample1 = { 6, 7, 9, 13, 19, 21, 22, 23, 24 };
530 final double[] smallSample2 = { 10, 11, 12, 16, 20, 27, 28, 32, 44, 54 };
531
532 assertEquals(0.105577085453247,
533 InferenceTestUtils.kolmogorovSmirnovTest(smallSample1, smallSample2, false), tol);
534 final double d = InferenceTestUtils.kolmogorovSmirnovStatistic(smallSample1, smallSample2);
535 assertEquals(0.5, d, tol);
536 assertEquals(0.105577085453247,
537 InferenceTestUtils.exactP(d, smallSample1.length,smallSample2.length, false), tol);
538 }
539 }