Review Board 1.7.22


HBASE-4071: Support minimum number of versions with TTL

Review Request #1582 - Created Aug. 18, 2011 and submitted

Lars Hofhansl
trunk
HBASE-4071
Reviewers
hbase
jgray, stack, tlipcon
hbase
Allow enforcing a minimum number of versions when TTL is enable for a store.
The GC logic for both versions and TTL is unified inside the ColumnTrackers.
Ran all tests. I get error (not failures) from two: TestDistributedLogSplitting and TestHTablePool. Both fail with or without my changes.
New tests: TestMinVersions.

Diff revision 2

This is not the most recent revision of the diff. The latest diff is revision 7. See what's changed.

1 2 3 4 5 6 7
1 2 3 4 5 6 7

  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java: Loading...
  13. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java: Loading...
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
Revision 1159317 New Change
[20] 84 lines
[+20] [+] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
85
  public static final String LENGTH = "LENGTH";
85
  public static final String LENGTH = "LENGTH";
86
  public static final String TTL = "TTL";
86
  public static final String TTL = "TTL";
87
  public static final String BLOOMFILTER = "BLOOMFILTER";
87
  public static final String BLOOMFILTER = "BLOOMFILTER";
88
  public static final String FOREVER = "FOREVER";
88
  public static final String FOREVER = "FOREVER";
89
  public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";
89
  public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";

    
   
90
  public static final String MIN_VERSIONS = "MIN_VERSIONS";
90

    
   
91

   
91
  /**
92
  /**
92
   * Default compression type.
93
   * Default compression type.
93
   */
94
   */
94
  public static final String DEFAULT_COMPRESSION =
95
  public static final String DEFAULT_COMPRESSION =
95
    Compression.Algorithm.NONE.getName();
96
    Compression.Algorithm.NONE.getName();
96

    
   
97

   
97
  /**
98
  /**
98
   * Default number of versions of a record to keep.
99
   * Default number of versions of a record to keep.
99
   */
100
   */
100
  public static final int DEFAULT_VERSIONS = 3;
101
  public static final int DEFAULT_VERSIONS = 3;
101

    
   
102

   

    
   
103
  /**

    
   
104
   * Default is not to keep a minimum of versions.

    
   
105
   */

    
   
106
  public static final int DEFAULT_MIN_VERSIONS = 0;

    
   
107

   
102
  /*
108
  /*
103
   * Cache here the HCD value.
109
   * Cache here the HCD value.
104
   * Question: its OK to cache since when we're reenable, we create a new HCD?
110
   * Question: its OK to cache since when we're reenable, we create a new HCD?
105
   */
111
   */
106
  private volatile Integer blocksize = null;
112
  private volatile Integer blocksize = null;
[+20] [20] 138 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
245
   */
251
   */
246
  public HColumnDescriptor(final byte [] familyName, final int maxVersions,
252
  public HColumnDescriptor(final byte [] familyName, final int maxVersions,
247
      final String compression, final boolean inMemory,
253
      final String compression, final boolean inMemory,
248
      final boolean blockCacheEnabled, final int blocksize,
254
      final boolean blockCacheEnabled, final int blocksize,
249
      final int timeToLive, final String bloomFilter, final int scope) {
255
      final int timeToLive, final String bloomFilter, final int scope) {

    
   
256
    this(familyName, DEFAULT_MIN_VERSIONS, maxVersions, compression, inMemory, blockCacheEnabled,

    
   
257
        blocksize, timeToLive, bloomFilter, scope);

    
   
258
  }

    
   
259

   

    
   
260
  /**

    
   
261
   * Constructor

    
   
262
   * @param familyName Column family name. Must be 'printable' -- digit or

    
   
263
   * letter -- and may not contain a <code>:<code>

    
   
264
   * @param maxVersions Maximum number of versions to keep

    
   
265
   * @param minVersions Minimum number of versions to keep

    
   
266
   * @param compression Compression type

    
   
267
   * @param inMemory If true, column data should be kept in an HRegionServer's

    
   
268
   * cache

    
   
269
   * @param blockCacheEnabled If true, MapFile blocks should be cached

    
   
270
   * @param blocksize Block size to use when writing out storefiles.  Use

    
   
271
   * smaller blocksizes for faster random-access at expense of larger indices

    
   
272
   * (more memory consumption).  Default is usually 64k.

    
   
273
   * @param timeToLive Time-to-live of cell contents, in seconds

    
   
274
   * (use HConstants.FOREVER for unlimited TTL)

    
   
275
   * @param bloomFilter Bloom filter type for this column

    
   
276
   * @param scope The scope tag for this column

    
   
277
   *

    
   
278
   * @throws IllegalArgumentException if passed a family name that is made of

    
   
279
   * other than 'word' characters: i.e. <code>[a-zA-Z_0-9]</code> or contains

    
   
280
   * a <code>:</code>

    
   
281
   * @throws IllegalArgumentException if the number of versions is &lt;= 0

    
   
282
   */

    
   
283
  public HColumnDescriptor(final byte [] familyName, final int minVersions, final int maxVersions,

    
   
284
      final String compression, final boolean inMemory,

    
   
285
      final boolean blockCacheEnabled, final int blocksize,

    
   
286
      final int timeToLive, final String bloomFilter, final int scope) {
250
    isLegalFamilyName(familyName);
287
    isLegalFamilyName(familyName);
251
    this.name = familyName;
288
    this.name = familyName;
252

    
   
289

   
253
    if (maxVersions <= 0) {
290
    if (maxVersions <= 0) {
254
      // TODO: Allow maxVersion of 0 to be the way you say "Keep all versions".
291
      // TODO: Allow maxVersion of 0 to be the way you say "Keep all versions".
255
      // Until there is support, consider 0 or < 0 -- a configuration error.
292
      // Until there is support, consider 0 or < 0 -- a configuration error.
256
      throw new IllegalArgumentException("Maximum versions must be positive");
293
      throw new IllegalArgumentException("Maximum versions must be positive");
257
    }
294
    }

    
   
295

   

    
   
296
    if (minVersions > 0) {

    
   
297
      if (timeToLive == HConstants.FOREVER) {

    
   
298
        throw new IllegalArgumentException("Minimum versions requires TTL to set.");

    
   
299
      }

    
   
300
      if (minVersions > maxVersions) {

    
   
301
        throw new IllegalArgumentException("Minimum versions must be <= maximum versions.");

    
   
302
      }

    
   
303
    }

    
   
304
    
258
    setMaxVersions(maxVersions);
305
    setMaxVersions(maxVersions);

    
   
306
    setMinVersions(minVersions);
259
    setInMemory(inMemory);
307
    setInMemory(inMemory);
260
    setBlockCacheEnabled(blockCacheEnabled);
308
    setBlockCacheEnabled(blockCacheEnabled);
261
    setTimeToLive(timeToLive);
309
    setTimeToLive(timeToLive);
262
    setCompressionType(Compression.Algorithm.
310
    setCompressionType(Compression.Algorithm.
263
      valueOf(compression.toUpperCase()));
311
      valueOf(compression.toUpperCase()));
[+20] [20] 230 lines
[+20] [+] public int getTimeToLive() {
494
   */
542
   */
495
  public void setTimeToLive(int timeToLive) {
543
  public void setTimeToLive(int timeToLive) {
496
    setValue(TTL, Integer.toString(timeToLive));
544
    setValue(TTL, Integer.toString(timeToLive));
497
  }
545
  }
498

    
   
546

   

    
   
547
  public int getMinVersions() {

    
   
548
    String value = getValue(MIN_VERSIONS);

    
   
549
    return (value != null)? Integer.valueOf(value).intValue(): 0;

    
   
550
  }

    
   
551

   

    
   
552
  public void setMinVersions(int minVersions) {

    
   
553
    setValue(MIN_VERSIONS, Integer.toString(minVersions));

    
   
554
  }

    
   
555

   
499
  /**
556
  /**
500
   * @return True if MapFile blocks should be cached.
557
   * @return True if MapFile blocks should be cached.
501
   */
558
   */
502
  public boolean isBlockCacheEnabled() {
559
  public boolean isBlockCacheEnabled() {
503
    String value = getValue(BLOCKCACHE);
560
    String value = getValue(BLOCKCACHE);
[+20] [20] 195 lines
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
Revision 1159317 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
New File
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
Revision 1159317 New Change
 
  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java: Loading...
  13. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java: Loading...