Review Board 1.7.22


HIVE-4957: Restrict number of bit vectors, to prevent out of Java heap memory

Review Request #14250 - Created Sept. 20, 2013 and updated

Shreepadma Venugopalan
trunk
HIVE-4957
Reviewers
hive
brocknoland
hive-git
Restricts the number of bit vectors used by Flajolet-Martin distinct value estimator to 1024.
Adds a new negative test case.
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java
Revision 63110bb New Change
[20] 474 lines
[+20] [+] public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
475
      if (myagg.firstItem) {
475
      if (myagg.firstItem) {
476
        int numVectors = 0;
476
        int numVectors = 0;
477
        if (!emptyTable) {
477
        if (!emptyTable) {
478
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
478
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
479
        }
479
        }

    
   
480
        if (numVectors > 1024) {

    
   
481
          throw new HiveException("The maximum allowed value for number of bit vectors " +

    
   
482
            " is 1024, but was passed " + numVectors + " bit vectors");

    
   
483
        }
480
        initNDVEstimator(myagg, numVectors);
484
        initNDVEstimator(myagg, numVectors);
481
        myagg.firstItem = false;
485
        myagg.firstItem = false;
482
        myagg.numBitVectors = numVectors;
486
        myagg.numBitVectors = numVectors;
483
      }
487
      }
484

    
   
488

   
[+20] [20] 286 lines
[+20] [+] public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
771
      if (myagg.firstItem) {
775
      if (myagg.firstItem) {
772
        int numVectors = 0;
776
        int numVectors = 0;
773
        if (!emptyTable) {
777
        if (!emptyTable) {
774
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
778
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
775
        }
779
        }

    
   
780

   

    
   
781
        if (numVectors > 1024) {

    
   
782
          throw new HiveException("The maximum allowed value for number of bit vectors " +

    
   
783
            " is 1024, but was passed " + numVectors + " bit vectors");

    
   
784
        }

    
   
785

   
776
        initNDVEstimator(myagg, numVectors);
786
        initNDVEstimator(myagg, numVectors);
777
        myagg.firstItem = false;
787
        myagg.firstItem = false;
778
        myagg.numBitVectors = numVectors;
788
        myagg.numBitVectors = numVectors;
779
      }
789
      }
780

    
   
790

   
[+20] [20] 298 lines
[+20] [+] public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
1079
      if (myagg.firstItem) {
1089
      if (myagg.firstItem) {
1080
        int numVectors = 0;
1090
        int numVectors = 0;
1081
        if (!emptyTable) {
1091
        if (!emptyTable) {
1082
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
1092
          numVectors = PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI);
1083
        }
1093
        }

    
   
1094

   

    
   
1095
        if (numVectors > 1024) {

    
   
1096
          throw new HiveException("The maximum allowed value for number of bit vectors " +

    
   
1097
            " is 1024, but was passed " + numVectors + " bit vectors");

    
   
1098
        }

    
   
1099

   
1084
        initNDVEstimator(myagg, numVectors);
1100
        initNDVEstimator(myagg, numVectors);
1085
        myagg.firstItem = false;
1101
        myagg.firstItem = false;
1086
        myagg.numBitVectors = numVectors;
1102
        myagg.numBitVectors = numVectors;
1087
      }
1103
      }
1088

    
   
1104

   
[+20] [20] 369 lines
ql/src/test/queries/clientnegative/compute_stats_long.q
New File
 
ql/src/test/results/clientnegative/compute_stats_long.q.out
New File
 
  1. ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java: Loading...
  2. ql/src/test/queries/clientnegative/compute_stats_long.q: Loading...
  3. ql/src/test/results/clientnegative/compute_stats_long.q.out: Loading...