Review Board 1.7.22


HBASE-2214 per scan max buffersize

Review Request #4726 - Created April 15, 2012 and updated

ferdy
0.94
HBASE-2214
Reviewers
hbase
tedyu
hbase
HBASE-2214 per scan max buffersize.
It works when running this test:



    new HBaseTestingUtility(conf).startMiniCluster();
 
    HBaseAdmin admin = new HBaseAdmin(conf);
    if (!admin.tableExists("test")) {
      HTableDescriptor tableDesc = new HTableDescriptor("test");
      tableDesc.addFamily(new HColumnDescriptor("fam"));
      admin.createTable(tableDesc);
    }
    
    
    HTable table = new HTable(conf, "test");
    Put put; 
    
    put = new Put(Bytes.toBytes("row1"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual1"),Bytes.toBytes("val1"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row2"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual2"),Bytes.toBytes("val2"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row3"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual3"),Bytes.toBytes("val3"));
    table.put(put);
    
    table.flushCommits();
    //put a logging statement to ClientScanner#next() to see the effect.
    {
      System.out.println("returns all rows at once because of the caching");
      Scan scan = new Scan();
      scan.setCaching(100);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }
    {
      System.out.println("returns one row at a time because of the maxResultSize");
      Scan scan = new Scan();
      scan.setCaching(100);
      scan.setMaxResultSize(1);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }



See output:

returns all rows at once because of the caching
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,499 DEBUG [main] client.ClientScanner(323): Rows returned 3
2012-04-25 22:18:47,502 DEBUG [main] client.ClientScanner(193): Finished with scanning at {NAME => 'test,,1335385126388.ed23a82f3d6ca2eab571918843796259.', STARTKEY => '', ENDKEY => '', ENCODED => ed23a82f3d6ca2eab571918843796259,}
returns one row at a time because of the maxResultSize
2012-04-25 22:18:47,504 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,505 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,514 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,517 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,522 DEBUG [main] client.ClientScanner(323): Rows returned 1
/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
Diff Revision 4 Diff Revision 5
[20] 317 lines
[+20] [+] public Result next() throws IOException {
318
          if (this.scanMetrics != null ) {
318
          if (this.scanMetrics != null ) {
319
            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext);
319
            this.scanMetrics.sumOfMillisSecBetweenNexts.inc(currentTime-lastNext);
320
          }
320
          }
321
          lastNext = currentTime;
321
          lastNext = currentTime;
322
          if (values != null && values.length > 0) {
322
          if (values != null && values.length > 0) {
323
            if (LOG.isDebugEnabled()) {LOG.debug("Rows returned " + values.length);}

   
324
            for (Result rs : values) {
323
            for (Result rs : values) {
325
              cache.add(rs);
324
              cache.add(rs);
326
              for (KeyValue kv : rs.raw()) {
325
              for (KeyValue kv : rs.raw()) {
327
                  remainingResultSize -= kv.heapSize();
326
                  remainingResultSize -= kv.heapSize();
328
              }
327
              }
[+20] [20] 63 lines
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Diff Revision 4 Diff Revision 5
 
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Diff Revision 4 Diff Revision 5
 
/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
Diff Revision 4 Diff Revision 5
 
/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServer.java
Diff Revision 4 Diff Revision 5
 
/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
Diff Revision 4 Diff Revision 5
 
  1. /src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java: Loading...
  2. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  3. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  4. /src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java: Loading...
  5. /src/main/java/org/apache/hadoop/hbase/regionserver/RegionServer.java: Loading...
  6. /src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java: Loading...