Review Board 1.7.22


HBASE-4536 - Allow CF to retain deleted rows

Review Request #2178 - Created Oct. 4, 2011 and submitted

Lars Hofhansl
trunk
HBASE-4536
Reviewers
hbase
jgray, tedyu
hbase
HBase timerange Gets and Scans allow to do "timetravel" in HBase. I.e. look at the state of the data at any point in the past, provided the data is still around.
This did not work for deletes, however. Deletes would always mask all puts in the past.
This change adds a flag that can be on HColumnDescriptor to enable retention of deleted rows.
These rows are still subject to TTL and/or VERSIONS.

This changes the following:
1. There is a new flag on HColumnDescriptor enabling that behavior.
2. Allow gets/scans with a timerange to retrieve rows hidden by a delete marker, if the timerange does not include the delete marker.
3. Do not unconditionally collect all deleted rows during a compaction.
4. Allow a "raw" Scan, which retrieves all delete markers and deleted rows.

The change is small'ish, but the logic is intricate, so please review carefully.
All tests pass now.

Diff revision 8

This is not the most recent revision of the diff. The latest diff is revision 12. See what's changed.

1 2 3 4 5 6 7 8 9 10 11 12
1 2 3 4 5 6 7 8 9 10 11 12

  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Scan.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java: Loading...
  13. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java: Loading...
  14. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java: Loading...
  15. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java: Loading...
  16. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java: Loading...
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
Revision 1183459 New Change
[20] 85 lines
[+20] [+] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
86
  public static final String TTL = "TTL";
86
  public static final String TTL = "TTL";
87
  public static final String BLOOMFILTER = "BLOOMFILTER";
87
  public static final String BLOOMFILTER = "BLOOMFILTER";
88
  public static final String FOREVER = "FOREVER";
88
  public static final String FOREVER = "FOREVER";
89
  public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";
89
  public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";
90
  public static final String MIN_VERSIONS = "MIN_VERSIONS";
90
  public static final String MIN_VERSIONS = "MIN_VERSIONS";

    
   
91
  public static final String KEEP_DELETED_ROWS = "KEEP_DELETED_ROWS";
91

    
   
92

   
92
  /**
93
  /**
93
   * Default compression type.
94
   * Default compression type.
94
   */
95
   */
95
  public static final String DEFAULT_COMPRESSION =
96
  public static final String DEFAULT_COMPRESSION =
[+20] [20] 19 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
115
   * Default setting for whether to serve from memory or not.
116
   * Default setting for whether to serve from memory or not.
116
   */
117
   */
117
  public static final boolean DEFAULT_IN_MEMORY = false;
118
  public static final boolean DEFAULT_IN_MEMORY = false;
118

    
   
119

   
119
  /**
120
  /**

    
   
121
   * Default setting for preventing deleted from being collected immediately.

    
   
122
   */

    
   
123
  public static final boolean DEFAULT_KEEP_DELETED = false;

    
   
124

   

    
   
125
  /**
120
   * Default setting for whether to use a block cache or not.
126
   * Default setting for whether to use a block cache or not.
121
   */
127
   */
122
  public static final boolean DEFAULT_BLOCKCACHE = true;
128
  public static final boolean DEFAULT_BLOCKCACHE = true;
123

    
   
129

   
124
  /**
130
  /**
[+20] [20] 24 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
149
      DEFAULT_VALUES.put(COMPRESSION, DEFAULT_COMPRESSION);
155
      DEFAULT_VALUES.put(COMPRESSION, DEFAULT_COMPRESSION);
150
      DEFAULT_VALUES.put(TTL, String.valueOf(DEFAULT_TTL));
156
      DEFAULT_VALUES.put(TTL, String.valueOf(DEFAULT_TTL));
151
      DEFAULT_VALUES.put(BLOCKSIZE, String.valueOf(DEFAULT_BLOCKSIZE));
157
      DEFAULT_VALUES.put(BLOCKSIZE, String.valueOf(DEFAULT_BLOCKSIZE));
152
      DEFAULT_VALUES.put(HConstants.IN_MEMORY, String.valueOf(DEFAULT_IN_MEMORY));
158
      DEFAULT_VALUES.put(HConstants.IN_MEMORY, String.valueOf(DEFAULT_IN_MEMORY));
153
      DEFAULT_VALUES.put(BLOCKCACHE, String.valueOf(DEFAULT_BLOCKCACHE));
159
      DEFAULT_VALUES.put(BLOCKCACHE, String.valueOf(DEFAULT_BLOCKCACHE));

    
   
160
      DEFAULT_VALUES.put(KEEP_DELETED_ROWS, String.valueOf(DEFAULT_KEEP_DELETED));
154
  }
161
  }
155

    
   
162

   
156
  // Column family name
163
  // Column family name
157
  private byte [] name;
164
  private byte [] name;
158

    
   
165

   
[+20] [20] 104 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
263
   */
270
   */
264
  public HColumnDescriptor(final byte [] familyName, final int maxVersions,
271
  public HColumnDescriptor(final byte [] familyName, final int maxVersions,
265
      final String compression, final boolean inMemory,
272
      final String compression, final boolean inMemory,
266
      final boolean blockCacheEnabled, final int blocksize,
273
      final boolean blockCacheEnabled, final int blocksize,
267
      final int timeToLive, final String bloomFilter, final int scope) {
274
      final int timeToLive, final String bloomFilter, final int scope) {
268
    this(familyName, DEFAULT_MIN_VERSIONS, maxVersions, compression, inMemory,
275
    this(familyName, DEFAULT_MIN_VERSIONS, maxVersions, DEFAULT_KEEP_DELETED,
269
        blockCacheEnabled, blocksize, timeToLive, bloomFilter, scope);
276
        compression, inMemory, blockCacheEnabled, blocksize, timeToLive,

    
   
277
        bloomFilter, scope);
270
  }
278
  }
271

    
   
279

   
272
  /**
280
  /**
273
   * Constructor
281
   * Constructor
274
   * @param familyName Column family name. Must be 'printable' -- digit or
282
   * @param familyName Column family name. Must be 'printable' -- digit or
[+20] [20] 15 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
290
   * @throws IllegalArgumentException if passed a family name that is made of
298
   * @throws IllegalArgumentException if passed a family name that is made of
291
   * other than 'word' characters: i.e. <code>[a-zA-Z_0-9]</code> or contains
299
   * other than 'word' characters: i.e. <code>[a-zA-Z_0-9]</code> or contains
292
   * a <code>:</code>
300
   * a <code>:</code>
293
   * @throws IllegalArgumentException if the number of versions is &lt;= 0
301
   * @throws IllegalArgumentException if the number of versions is &lt;= 0
294
   */
302
   */
295
  public HColumnDescriptor(final byte [] familyName, final int minVersions,
303
  public HColumnDescriptor(final byte[] familyName, final int minVersions,
296
      final int maxVersions, final String compression, final boolean inMemory,
304
      final int maxVersions, final boolean keepDeletedRows,

    
   
305
      final String compression, final boolean inMemory,
297
      final boolean blockCacheEnabled, final int blocksize,
306
      final boolean blockCacheEnabled, final int blocksize,
298
      final int timeToLive, final String bloomFilter, final int scope) {
307
      final int timeToLive, final String bloomFilter, final int scope) {
299
    isLegalFamilyName(familyName);
308
    isLegalFamilyName(familyName);
300
    this.name = familyName;
309
    this.name = familyName;
301

    
   
310

   
[+20] [20] 5 lines
[+20] public class HColumnDescriptor implements WritableComparable<HColumnDescriptor> {
307

    
   
316

   
308
    if (minVersions > 0) {
317
    if (minVersions > 0) {
309
      if (timeToLive == HConstants.FOREVER) {
318
      if (timeToLive == HConstants.FOREVER) {
310
        throw new IllegalArgumentException("Minimum versions requires TTL.");
319
        throw new IllegalArgumentException("Minimum versions requires TTL.");
311
      }
320
      }
312
      if (minVersions > maxVersions) {
321
      if (minVersions >= maxVersions) {
313
        throw new IllegalArgumentException("Minimum versions must be <= "+
322
        throw new IllegalArgumentException("Minimum versions must be < "
314
            "maximum versions.");
323
            + "maximum versions.");
315
      }
324
      }
316
    }
325
    }
317

    
   
326

   
318
    setMaxVersions(maxVersions);
327
    setMaxVersions(maxVersions);
319
    setMinVersions(minVersions);
328
    setMinVersions(minVersions);

    
   
329
    setKeepDeletedRows(keepDeletedRows);
320
    setInMemory(inMemory);
330
    setInMemory(inMemory);
321
    setBlockCacheEnabled(blockCacheEnabled);
331
    setBlockCacheEnabled(blockCacheEnabled);
322
    setTimeToLive(timeToLive);
332
    setTimeToLive(timeToLive);
323
    setCompressionType(Compression.Algorithm.
333
    setCompressionType(Compression.Algorithm.
324
      valueOf(compression.toUpperCase()));
334
      valueOf(compression.toUpperCase()));
[+20] [20] 215 lines
[+20] [+] public boolean isInMemory() {
540
   */
550
   */
541
  public void setInMemory(boolean inMemory) {
551
  public void setInMemory(boolean inMemory) {
542
    setValue(HConstants.IN_MEMORY, Boolean.toString(inMemory));
552
    setValue(HConstants.IN_MEMORY, Boolean.toString(inMemory));
543
  }
553
  }
544

    
   
554

   

    
   
555
  public boolean getKeepDeletedRows() {

    
   
556
    String value = getValue(KEEP_DELETED_ROWS);

    
   
557
    if (value != null) {

    
   
558
      return Boolean.valueOf(value).booleanValue();

    
   
559
    }

    
   
560
    return DEFAULT_KEEP_DELETED;

    
   
561
  }

    
   
562

   

    
   
563
  /**

    
   
564
   * @param keepDeletedRows True if deleted rows should not be collected

    
   
565
   * immediately.

    
   
566
   */

    
   
567
  public void setKeepDeletedRows(boolean keepDeletedRows) {

    
   
568
    setValue(KEEP_DELETED_ROWS, Boolean.toString(keepDeletedRows));

    
   
569
  }

    
   
570

   
545
  /**
571
  /**
546
   * @return Time-to-live of cell contents, in seconds.
572
   * @return Time-to-live of cell contents, in seconds.
547
   */
573
   */
548
  public int getTimeToLive() {
574
  public int getTimeToLive() {
549
    String value = getValue(TTL);
575
    String value = getValue(TTL);
[+20] [20] 252 lines
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Scan.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java
New File
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java
Revision 1183459 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
Revision 1183459 New Change
 
  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Scan.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/main/ruby/hbase/admin.rb: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java: Loading...
  13. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java: Loading...
  14. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java: Loading...
  15. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java: Loading...
  16. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java: Loading...