Review Board 1.7.22


HBASE-2214 per scan max buffersize

Review Request #4726 - Created April 15, 2012 and updated

ferdy
0.94
HBASE-2214
Reviewers
hbase
tedyu
hbase
HBASE-2214 per scan max buffersize.
It works when running this test:



    new HBaseTestingUtility(conf).startMiniCluster();
 
    HBaseAdmin admin = new HBaseAdmin(conf);
    if (!admin.tableExists("test")) {
      HTableDescriptor tableDesc = new HTableDescriptor("test");
      tableDesc.addFamily(new HColumnDescriptor("fam"));
      admin.createTable(tableDesc);
    }
    
    
    HTable table = new HTable(conf, "test");
    Put put; 
    
    put = new Put(Bytes.toBytes("row1"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual1"),Bytes.toBytes("val1"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row2"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual2"),Bytes.toBytes("val2"));
    table.put(put);
    
    put = new Put(Bytes.toBytes("row3"));
    put.add(Bytes.toBytes("fam"),Bytes.toBytes("qual3"),Bytes.toBytes("val3"));
    table.put(put);
    
    table.flushCommits();
    //put a logging statement to ClientScanner#next() to see the effect.
    {
      System.out.println("returns all rows at once because of the caching");
      Scan scan = new Scan();
      scan.setCaching(100);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }
    {
      System.out.println("returns one row at a time because of the maxResultSize");
      Scan scan = new Scan();
      scan.setCaching(100);
      scan.setMaxResultSize(1);
      ResultScanner scanner = table.getScanner(scan);
      scanner.next(100);
    }



See output:

returns all rows at once because of the caching
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,494 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,499 DEBUG [main] client.ClientScanner(323): Rows returned 3
2012-04-25 22:18:47,502 DEBUG [main] client.ClientScanner(193): Finished with scanning at {NAME => 'test,,1335385126388.ed23a82f3d6ca2eab571918843796259.', STARTKEY => '', ENDKEY => '', ENCODED => ed23a82f3d6ca2eab571918843796259,}
returns one row at a time because of the maxResultSize
2012-04-25 22:18:47,504 DEBUG [main] client.ClientScanner(94): Creating scanner over test starting at key ''
2012-04-25 22:18:47,505 DEBUG [main] client.ClientScanner(206): Advancing internal scanner to startKey at ''
2012-04-25 22:18:47,514 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,517 DEBUG [main] client.ClientScanner(323): Rows returned 1
2012-04-25 22:18:47,522 DEBUG [main] client.ClientScanner(323): Rows returned 1
/src/main/java/org/apache/hadoop/hbase/client/Scan.java
Revision 1326084 New Change
[20] 50 lines
[+20]
51
 * iterate over all rows.
51
 * iterate over all rows.
52
 * <p>
52
 * <p>
53
 * To scan everything for each row, instantiate a Scan object.
53
 * To scan everything for each row, instantiate a Scan object.
54
 * <p>
54
 * <p>
55
 * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
55
 * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
56
 * If caching is NOT set, we will use the caching value of the hosting
56
 * If caching is NOT set, we will use the caching value of the hosting {@link HTable}.  See
57
 * {@link HTable}.  See {@link HTable#setScannerCaching(int)}.
57
 * {@link HTable#setScannerCaching(int)}. In addition to row caching, it is possible to specify a

    
   
58
 * maximum buffer size, using {@link #setMaxBufferSize(long) setMaxBufferSize}. When both are used,

    
   
59
 * single server requests are limited by either number of rows or maximum buffer size, whichever

    
   
60
 * limit comes first.
58
 * <p>
61
 * <p>
59
 * To further define the scope of what to get when scanning, perform additional
62
 * To further define the scope of what to get when scanning, perform additional
60
 * methods as outlined below.
63
 * methods as outlined below.
61
 * <p>
64
 * <p>
62
 * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily}
65
 * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily}
[+20] [20] 23 lines
[+20]
86
@InterfaceStability.Stable
89
@InterfaceStability.Stable
87
public class Scan extends OperationWithAttributes implements Writable {
90
public class Scan extends OperationWithAttributes implements Writable {
88
  private static final String RAW_ATTR = "_raw_";
91
  private static final String RAW_ATTR = "_raw_";
89
  private static final String ISOLATION_LEVEL = "_isolationlevel_";
92
  private static final String ISOLATION_LEVEL = "_isolationlevel_";
90

    
   
93

   
91
  private static final byte SCAN_VERSION = (byte)2;
94
  private static final byte SCAN_VERSION = (byte)3;
92
  private byte [] startRow = HConstants.EMPTY_START_ROW;
95
  private byte [] startRow = HConstants.EMPTY_START_ROW;
93
  private byte [] stopRow  = HConstants.EMPTY_END_ROW;
96
  private byte [] stopRow  = HConstants.EMPTY_END_ROW;
94
  private int maxVersions = 1;
97
  private int maxVersions = 1;
95
  private int batch = -1;
98
  private int batch = -1;
96
  // If application wants to collect scan metrics, it needs to
99
  // If application wants to collect scan metrics, it needs to
[+20] [20] 5 lines
[+20] public class Scan extends OperationWithAttributes implements Writable {
102

    
   
105

   
103
  /*
106
  /*
104
   * -1 means no caching
107
   * -1 means no caching
105
   */
108
   */
106
  private int caching = -1;
109
  private int caching = -1;

    
   
110
  private long maxBufferSize = -1;
107
  private boolean cacheBlocks = true;
111
  private boolean cacheBlocks = true;
108
  private Filter filter = null;
112
  private Filter filter = null;
109
  private TimeRange tr = new TimeRange();
113
  private TimeRange tr = new TimeRange();
110
  private Map<byte [], NavigableSet<byte []>> familyMap =
114
  private Map<byte [], NavigableSet<byte []>> familyMap =
111
    new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
115
    new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
[+20] [20] 39 lines
[+20] public class Scan extends OperationWithAttributes implements Writable {
151
    startRow = scan.getStartRow();
155
    startRow = scan.getStartRow();
152
    stopRow  = scan.getStopRow();
156
    stopRow  = scan.getStopRow();
153
    maxVersions = scan.getMaxVersions();
157
    maxVersions = scan.getMaxVersions();
154
    batch = scan.getBatch();
158
    batch = scan.getBatch();
155
    caching = scan.getCaching();
159
    caching = scan.getCaching();

    
   
160
    maxBufferSize = scan.getMaxBufferSize();
156
    cacheBlocks = scan.getCacheBlocks();
161
    cacheBlocks = scan.getCacheBlocks();
157
    filter = scan.getFilter(); // clone?
162
    filter = scan.getFilter(); // clone?
158
    TimeRange ctr = scan.getTimeRange();
163
    TimeRange ctr = scan.getTimeRange();
159
    tr = new TimeRange(ctr.getMin(), ctr.getMax());
164
    tr = new TimeRange(ctr.getMin(), ctr.getMax());
160
    Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap();
165
    Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap();
[+20] [20] 164 lines
[+20] [+] public void setBatch(int batch) {
325
  public void setCaching(int caching) {
330
  public void setCaching(int caching) {
326
    this.caching = caching;
331
    this.caching = caching;
327
  }
332
  }
328

    
   
333

   
329
  /**
334
  /**

    
   
335
   * @return the maximum buffer size in bytes. See {@link #setMaxBufferSize(long)}

    
   
336
   */

    
   
337
  public long getMaxBufferSize() {

    
   
338
    return maxBufferSize;

    
   
339
  }

    
   
340

   

    
   
341
  /**

    
   
342
   * Set the maximum buffer size. The default is -1; this means that no specific

    
   
343
   * maximum buffer size will be set for this scan, and the global configured

    
   
344
   * value will be used instead. (Defaults to unlimited).

    
   
345
   *

    
   
346
   * @param maxBufferSize The maximum buffer size in bytes.

    
   
347
   */

    
   
348
  public void setMaxBufferSize(long maxBufferSize) {

    
   
349
    this.maxBufferSize = maxBufferSize;

    
   
350
  }

    
   
351

   

    
   
352
  /**
330
   * Apply the specified server-side filter when performing the Scan.
353
   * Apply the specified server-side filter when performing the Scan.
331
   * @param filter filter to run on the server
354
   * @param filter filter to run on the server
332
   * @return this
355
   * @return this
333
   */
356
   */
334
  public Scan setFilter(Filter filter) {
357
  public Scan setFilter(Filter filter) {
[+20] [20] 167 lines
[+20] [+] public boolean getCacheBlocks() {
502
    map.put("startRow", Bytes.toStringBinary(this.startRow));
525
    map.put("startRow", Bytes.toStringBinary(this.startRow));
503
    map.put("stopRow", Bytes.toStringBinary(this.stopRow));
526
    map.put("stopRow", Bytes.toStringBinary(this.stopRow));
504
    map.put("maxVersions", this.maxVersions);
527
    map.put("maxVersions", this.maxVersions);
505
    map.put("batch", this.batch);
528
    map.put("batch", this.batch);
506
    map.put("caching", this.caching);
529
    map.put("caching", this.caching);

    
   
530
    map.put("maxBufferSize", this.maxBufferSize);
507
    map.put("cacheBlocks", this.cacheBlocks);
531
    map.put("cacheBlocks", this.cacheBlocks);
508
    List<Long> timeRange = new ArrayList<Long>();
532
    List<Long> timeRange = new ArrayList<Long>();
509
    timeRange.add(this.tr.getMin());
533
    timeRange.add(this.tr.getMin());
510
    timeRange.add(this.tr.getMax());
534
    timeRange.add(this.tr.getMax());
511
    map.put("timeRange", timeRange);
535
    map.put("timeRange", timeRange);
[+20] [20] 72 lines
[+20] [+] public void readFields(final DataInput in)
584
    }
608
    }
585

    
   
609

   
586
    if (version > 1) {
610
    if (version > 1) {
587
      readAttributes(in);
611
      readAttributes(in);
588
    }
612
    }

    
   
613
    if (version > 2) {

    
   
614
      this.maxBufferSize = in.readLong();

    
   
615
    }
589
  }
616
  }
590

    
   
617

   
591
  public void write(final DataOutput out)
618
  public void write(final DataOutput out)
592
  throws IOException {
619
  throws IOException {
593
    out.writeByte(SCAN_VERSION);
620
    out.writeByte(SCAN_VERSION);
[+20] [20] 23 lines
[+20] public void write(final DataOutput out)
617
      } else {
644
      } else {
618
        out.writeInt(0);
645
        out.writeInt(0);
619
      }
646
      }
620
    }
647
    }
621
    writeAttributes(out);
648
    writeAttributes(out);

    
   
649
    out.writeLong(maxBufferSize);
622
  }
650
  }
623

    
   
651

   
624
  /**
652
  /**
625
   * Enable/disable "raw" mode for this scan.
653
   * Enable/disable "raw" mode for this scan.
626
   * If "raw" is enabled the scan will return all
654
   * If "raw" is enabled the scan will return all
[+20] [20] 45 lines
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Revision 1326084 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Revision 1326084 New Change
 
/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
Revision 1326084 New Change
 
/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
Revision 1326084 New Change
 
  1. /src/main/java/org/apache/hadoop/hbase/client/Scan.java: Loading...
  2. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  3. /src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  4. /src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java: Loading...
  5. /src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java: Loading...