Review Board 1.7.22


HBASE-2214 per scan max buffersize

Review Request #4726 - Created April 15, 2012 and updated

ferdy
0.94
HBASE-2214
Reviewers
hbase
tedyu
hbase
HBASE-2214 per scan max buffersize.
It works when running this test:



    new HBaseTestingUtility(conf).startMiniCluster();
 
    HBaseAdmin admin = new HBaseAdmin(conf);
    if (!admin.tableExists("test")) {
      HTableDescriptor tableDesc = new HTableDescriptor("test");
      tableDesc.addFamily(new HColumnDescriptor("fam"));
      admin.createTable(tableDesc);
    }
    
    
    HTable table = new HTable(conf, "test");
    Put put; 
    
    put = new Put(Bytes.toBytes("row1"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual1"),Bytes.toBytes("val1"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row2"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual2"),Bytes.toBytes("val2"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row3"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual3"),Bytes.toBytes("val3"));
    table.put(put);
    
    table.flushCommits();
    //put a logging statement to ClientScanner#next() to see the effect.
    {
      System.out.println("returns all rows at once because of the caching");
      Scan scan = new Scan();
      scan.setCaching(100);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }
    {
      System.out.println("returns one row at a time because of the maxResultSize");
      Scan scan = new Scan();
      scan.setCaching(100);
      scan.setMaxResultSize(1);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }



See output:

returns all rows at once because of the caching
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,499 DEBUG [main] client.ClientScanner(323): Rows returned 3
2012-04-25 22:18:47,502 DEBUG [main] client.ClientScanner(193): Finished with scanning at {NAME => 'test,,1335385126388.ed23a82f3d6ca2eab571918843796259.', STARTKEY => '', ENDKEY => '', ENCODED => ed23a82f3d6ca2eab571918843796259,}
returns one row at a time because of the maxResultSize
2012-04-25 22:18:47,504 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,505 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,514 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,517 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,522 DEBUG [main] client.ClientScanner(323): Rows returned 1
/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
Revision 1330504 New Change
[20] 96 lines
[+20] [+] public class ClientScanner extends AbstractClientScanner {
97
      }
97
      }
98
      this.scan = scan;
98
      this.scan = scan;
99
      this.tableName = tableName;
99
      this.tableName = tableName;
100
      this.lastNext = System.currentTimeMillis();
100
      this.lastNext = System.currentTimeMillis();
101
      this.connection = connection;
101
      this.connection = connection;

    
   
102
      if (scan.getMaxResultSize() > -1) {

    
   
103
        this.maxScannerResultSize = scan.getMaxResultSize();

    
   
104
      } else {
102
      this.maxScannerResultSize = conf.getLong(
105
        this.maxScannerResultSize = conf.getLong(
103
          HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
106
          HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
104
          HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
107
          HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);

    
   
108
      }
105
      this.scannerTimeout = (int) conf.getLong(
109
      this.scannerTimeout = (int) conf.getLong(
106
          HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
110
          HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
107
          HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
111
          HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
108

    
   
112

   
109
      // check if application wants to collect scan metrics
113
      // check if application wants to collect scan metrics
[+20] [20] 204 lines
[+20] [+] public Result next() throws IOException {
314
          if (this.scanMetrics != null ) {
318
          if (this.scanMetrics != null ) {
315
            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext);
319
            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext);
316
          }
320
          }
317
          lastNext = currentTime;
321
          lastNext = currentTime;
318
          if (values != null && values.length > 0) {
322
          if (values != null && values.length > 0) {

    
   
323
            if (LOG.isDebugEnabled()) {LOG.debug("Rows returned " + values.length);}
319
            for (Result rs : values) {
324
            for (Result rs : values) {
320
              cache.add(rs);
325
              cache.add(rs);
321
              for (KeyValue kv : rs.raw()) {
326
              for (KeyValue kv : rs.raw()) {
322
                  remainingResultSize -= kv.heapSize();
327
                  remainingResultSize -= kv.heapSize();
323
              }
328
              }
[+20] [20] 63 lines
/src/main/java/org/apache/hadoop/hbase/client/Scan.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/generated/AdminProtos.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
Revision 1330504 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServer.java
Revision 1330504 New Change
 
/src/main/protobuf/Client.proto
Revision 1330504 New Change
 
/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
Revision 1330504 New Change
 
  1. /src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java: Loading...
  2. /src/main/java/org/apache/hadoop/hbase/client/Scan.java: Loading...
  3. /src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java: Loading...
  4. /src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java: Loading...
  5. /src/main/java/org/apache/hadoop/hbase/protobuf/generated/AdminProtos.java: Loading...
  6. /src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java: Loading...
  7. /src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java: Loading...
  8. /src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java: Loading...
  9. /src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java: Loading...
  10. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  11. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  12. /src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java: Loading...
  13. /src/main/java/org/apache/hadoop/hbase/regionserver/RegionServer.java: Loading...
  14. /src/main/protobuf/Client.proto: Loading...
  15. /src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java: Loading...