Review Board 1.7.22


HBASE-5547: Don't delete HFiles when in backup mode.

Review Request #4633 - Created April 4, 2012 and submitted

Jesse Yates
HBASE-5547
Reviewers
hbase
lhofhansl, stack
hbase-git
Essentially, whenever an hfile would be deleted, it is instead moved to the archive directory. In this impl, the archive directory is on a per table basis, but defaults to '.archive'. Removing hfiles occurs in three places - compaction, merge and catalog janitor. The former and two latter are distinctly different code paths, but but did pull out some similarities. The latter two end up calling the same method, so there should be a reasonable amount of overlap.

Implementation wise: 
    Updated the HMasterInterface to pass the calls onto the zookeeper.
    Added a zk listener to handle updates from the master to the RS to backup.
    Added a utility for removing files and finding archive directories
    Added tests for the regionserver and catalogjanitor approaches.
    Added creation of manager in regionserver.
Added two tests for the separate cases - archiving via the regionserver and for the catalog tracker. Former runs in a mini cluster and also touches the changes to HMasterInterface and zookeeper.

Diff revision 3

This is not the most recent revision of the diff. The latest diff is revision 15. See what's changed.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

  1. src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveMonitor.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveTracker.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java: Loading...
  5. src/main/java/org/apache/hadoop/hbase/client/HFileArchiveManager.java: Loading...
  6. src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java: Loading...
  7. src/main/java/org/apache/hadoop/hbase/master/HMaster.java: Loading...
  8. src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  9. src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  10. src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerHFileTableArchiveTracker.java: Loading...
  11. src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java: Loading...
  12. src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java: Loading...
  13. src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  14. src/main/java/org/apache/hadoop/hbase/util/HFileArchiveCleanup.java: Loading...
  15. src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java: Loading...
  16. src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java: Loading...
  17. src/main/resources/hbase-default.xml: Loading...
  18. src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java: Loading...
  19. src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java: Loading...
  20. src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionHFileArchiving.java: Loading...
  21. src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java: Loading...
  22. src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java: Loading...
  23. src/test/java/org/apache/hadoop/hbase/util/TestHFileArchivingCleanup.java: Loading...
src/main/java/org/apache/hadoop/hbase/HConstants.java
Revision a9d80a0 New Change
[20] 671 lines
[+20]
672
/** Region in Transition metrics threshold time */
672
  /** Region in Transition metrics threshold time */
673
  public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold";
673
  public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold";
674

    
   
674

   
675
  public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
675
  public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
676

    
   
676

   

    
   
677
  /** Configuration key for the directory to backup HFiles for a table */

    
   
678
  public static final String HFILE_ARCHIVE_DIRECTORY = "hbase.table.archive.directory";

    
   
679
  public static final String HFILE_ARCHIVE_ZNODE_PARENT = "hfilearchive";

    
   
680

   
677
  private HConstants() {
681
  private HConstants() {
678
    // Can't be instantiated with this ctor.
682
    // Can't be instantiated with this ctor.
679
  }
683
  }
680
}
684
}
src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveMonitor.java
New File
 
src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveTracker.java
New File
 
src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
Revision ee16e72 New Change
 
src/main/java/org/apache/hadoop/hbase/client/HFileArchiveManager.java
New File
 
src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
Revision 79d5fdd New Change
 
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Revision d47b83a New Change
 
src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Revision 7858846 New Change
 
src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Revision 61a5988 New Change
 
src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerHFileTableArchiveTracker.java
New File
 
src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
Revision 6884d53 New Change
 
src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
Revision ea12da4 New Change
 
src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Revision bf1618e New Change
 
src/main/java/org/apache/hadoop/hbase/util/HFileArchiveCleanup.java
New File
 
src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
New File
 
src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
Revision 4fc105f New Change
 
src/main/resources/hbase-default.xml
Revision f54b345 New Change
 
src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
Revision a59e152 New Change
 
src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Revision cedf31e New Change
 
src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionHFileArchiving.java
New File
 
src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
New File
 
src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
Revision 7d02759 New Change
 
src/test/java/org/apache/hadoop/hbase/util/TestHFileArchivingCleanup.java
New File
 
  1. src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveMonitor.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/backup/HFileArchiveTracker.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java: Loading...
  5. src/main/java/org/apache/hadoop/hbase/client/HFileArchiveManager.java: Loading...
  6. src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java: Loading...
  7. src/main/java/org/apache/hadoop/hbase/master/HMaster.java: Loading...
  8. src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  9. src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  10. src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerHFileTableArchiveTracker.java: Loading...
  11. src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java: Loading...
  12. src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java: Loading...
  13. src/main/java/org/apache/hadoop/hbase/regionserver/Store.java: Loading...
  14. src/main/java/org/apache/hadoop/hbase/util/HFileArchiveCleanup.java: Loading...
  15. src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java: Loading...
  16. src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java: Loading...
  17. src/main/resources/hbase-default.xml: Loading...
  18. src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java: Loading...
  19. src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java: Loading...
  20. src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionHFileArchiving.java: Loading...
  21. src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java: Loading...
  22. src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java: Loading...
  23. src/test/java/org/apache/hadoop/hbase/util/TestHFileArchivingCleanup.java: Loading...