Review Board 1.7.22


HBASE-5719 Enhance hbck to sideline overlapped mega regions

Review Request #4649 - Created April 4, 2012 and submitted

Jimmy Xiang
trunk
HBASE-5719
Reviewers
hbase
jmhsieh
hbase-git
Make it configurable to sideline some regions in big overlapped groups which hbck doesn't handle currently.

The regions chose to sideline are those which overlap with most other regions.
mvn -PlocalTests -Dtest=TestHBaseFsck* clean test

Also tested in real system to fix inconsistencies.

Diff revision 2

This is not the most recent revision of the diff. The latest diff is revision 3. See what's changed.

1 2 3
1 2 3

  1. src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java: Loading...
  3. src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java: Loading...
src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
Revision 54f9b21 New Change
[20] 19 lines
[+20]
20
import java.io.IOException;
20
import java.io.IOException;
21
import java.util.ArrayList;
21
import java.util.ArrayList;
22
import java.util.Collection;
22
import java.util.Collection;
23
import java.util.Collections;
23
import java.util.Collections;
24
import java.util.Comparator;
24
import java.util.Comparator;

    
   
25
import java.util.HashMap;
25
import java.util.HashSet;
26
import java.util.HashSet;
26
import java.util.List;
27
import java.util.List;

    
   
28
import java.util.Map;
27
import java.util.Map.Entry;
29
import java.util.Map.Entry;
28
import java.util.Set;
30
import java.util.Set;
29
import java.util.SortedSet;
31
import java.util.SortedSet;
30
import java.util.TreeMap;
32
import java.util.TreeMap;
31
import java.util.TreeSet;
33
import java.util.TreeSet;
[+20] [20] 43 lines
[+20]
75
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
77
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
76
import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
78
import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
77
import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
79
import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
78
import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
80
import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
79
import org.apache.hadoop.hbase.zookeeper.ZKTable;
81
import org.apache.hadoop.hbase.zookeeper.ZKTable;
80
import org.apache.hadoop.hbase.zookeeper.ZKUtil;

   
81
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
82
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
82
import org.apache.zookeeper.KeeperException;
83
import org.apache.zookeeper.KeeperException;
83

    
   
84

   
84
import com.google.common.base.Joiner;
85
import com.google.common.base.Joiner;
85
import com.google.common.base.Preconditions;
86
import com.google.common.base.Preconditions;
[+20] [20] 52 lines
[+20]
138
  public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
139
  public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
139
  public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
140
  public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
140
  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
141
  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
141
  private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
142
  private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
142
  private static boolean rsSupportsOffline = true;
143
  private static boolean rsSupportsOffline = true;

    
   
144
  private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
143
  private static final int DEFAULT_MAX_MERGE = 5;
145
  private static final int DEFAULT_MAX_MERGE = 5;
144

    
   
146

   
145
  /**********************
147
  /**********************
146
   * Internal resources
148
   * Internal resources
147
   **********************/
149
   **********************/
148
  private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
150
  private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
149
  private Configuration conf;
151
  private Configuration conf;
150
  private ClusterStatus status;
152
  private ClusterStatus status;
151
  private HConnection connection;
153
  private HConnection connection;
152
  private HBaseAdmin admin;
154
  private HBaseAdmin admin;
153
  private HTable meta;
155
  private HTable meta;
154
  private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
156
  private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
155
  private int numThreads = MAX_NUM_THREADS;

   
156
  private long startMillis = System.currentTimeMillis();
157
  private long startMillis = System.currentTimeMillis();
157

    
   
158

   
158
  /***********
159
  /***********
159
   * Options
160
   * Options
160
   ***********/
161
   ***********/
[+20] [20] 6 lines
[+20]
167
  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
168
  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
168

    
   
169

   
169
  // limit fixes to listed tables, if empty atttempt to fix all
170
  // limit fixes to listed tables, if empty atttempt to fix all
170
  private List<byte[]> tablesToFix = new ArrayList<byte[]>();
171
  private List<byte[]> tablesToFix = new ArrayList<byte[]>();
171
  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
172
  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge

    
   
173
  private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline

    
   
174
  private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
172

    
   
175

   
173
  private boolean rerun = false; // if we tried to fix something, rerun hbck
176
  private boolean rerun = false; // if we tried to fix something, rerun hbck
174
  private static boolean summary = false; // if we want to print less output
177
  private static boolean summary = false; // if we want to print less output
175
  private boolean checkMetaOnly = false;
178
  private boolean checkMetaOnly = false;
176

    
   
179

   
[+20] [20] 35 lines
[+20]
212
   */
215
   */
213
  public HBaseFsck(Configuration conf) throws MasterNotRunningException,
216
  public HBaseFsck(Configuration conf) throws MasterNotRunningException,
214
      ZooKeeperConnectionException, IOException {
217
      ZooKeeperConnectionException, IOException {
215
    this.conf = conf;
218
    this.conf = conf;
216

    
   
219

   
217
    int numThreads = conf.getInt("hbasefsck.numthreads", Integer.MAX_VALUE);
220
    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
218
    executor = new ThreadPoolExecutor(1, numThreads,
221
    executor = new ThreadPoolExecutor(1, numThreads,
219
        THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
222
        THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
220
        new SynchronousQueue<Runnable>());
223
        new SynchronousQueue<Runnable>());
221
    executor.allowCoreThreadTimeOut(true);
224
    executor.allowCoreThreadTimeOut(true);
222
  }
225
  }
[+20] [20] 607 lines
[+20] [+] private Path getSidelineDir() throws IOException {
830
  }
833
  }
831

    
   
834

   
832
  /**
835
  /**
833
   * Sideline a region dir (instead of deleting it)
836
   * Sideline a region dir (instead of deleting it)
834
   */
837
   */
835
  void sidelineRegionDir(FileSystem fs, HbckInfo hi)
838
  Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
836
    throws IOException {
839
    throws IOException {
837
    String tableName = Bytes.toString(hi.getTableName());
840
    String tableName = Bytes.toString(hi.getTableName());
838
    Path regionDir = hi.getHdfsRegionDir();
841
    Path regionDir = hi.getHdfsRegionDir();
839

    
   
842

   
840
    if (!fs.exists(regionDir)) {
843
    if (!fs.exists(regionDir)) {
841
      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
844
      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
842
      return;
845
      return null;
843
    }
846
    }
844

    
   
847

   
845
    Path sidelineTableDir= new Path(getSidelineDir(), tableName);
848
    Path sidelineTableDir= new Path(getSidelineDir(), tableName);
846
    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
849
    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
847
    fs.mkdirs(sidelineRegionDir);
850
    fs.mkdirs(sidelineRegionDir);
[+20] [20] 22 lines
[+20] private Path getSidelineDir() throws IOException {
870
        LOG.info("Sidelining files from " + src + " into containing region " + dst);
873
        LOG.info("Sidelining files from " + src + " into containing region " + dst);
871
        // FileSystem.rename is inconsistent with directories -- if the
874
        // FileSystem.rename is inconsistent with directories -- if the
872
        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
875
        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
873
        // it moves the src into the dst dir resulting in (foo/a/b).  If
876
        // it moves the src into the dst dir resulting in (foo/a/b).  If
874
        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
877
        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
875
        for (FileStatus hfile : fs.listStatus(src)) {
878
        FileStatus[] hfiles = fs.listStatus(src);

    
   
879
        if (hfiles != null && hfiles.length > 0) {

    
   
880
          for (FileStatus hfile : hfiles) {
876
          success = fs.rename(hfile.getPath(), dst);
881
            success = fs.rename(hfile.getPath(), dst);
877
          if (!success) {
882
            if (!success) {
878
            String msg = "Unable to rename file " + src +  " to " + dst;
883
              String msg = "Unable to rename file " + src +  " to " + dst;
879
            LOG.error(msg);
884
              LOG.error(msg);
880
            throw new IOException(msg);
885
              throw new IOException(msg);
881
          }
886
            }
882
        }
887
          }

    
   
888
        }
883
        LOG.debug("Sideline directory contents:");
889
        LOG.debug("Sideline directory contents:");
884
        debugLsr(sidelineRegionDir);
890
        debugLsr(sidelineRegionDir);
885
      }
891
      }
886
    }
892
    }
887

    
   
893

   
888
    LOG.info("Removing old region dir: " + regionDir);
894
    LOG.info("Removing old region dir: " + regionDir);
889
    success = fs.delete(regionDir, true);
895
    success = fs.delete(regionDir, true);
890
    if (!success) {
896
    if (!success) {
891
      String msg = "Unable to delete dir " + regionDir;
897
      String msg = "Unable to delete dir " + regionDir;
892
      LOG.error(msg);
898
      LOG.error(msg);
893
      throw new IOException(msg);
899
      throw new IOException(msg);
894
    }
900
    }

    
   
901
    return sidelineRegionDir;
895
  }
902
  }
896

    
   
903

   
897
  /**
904
  /**
898
   * Side line an entire table.
905
   * Side line an entire table.
899
   */
906
   */
[+20] [20] 635 lines
[+20] [+] public class TableInfo {
1535
    TreeSet <ServerName> deployedOn;
1542
    TreeSet <ServerName> deployedOn;
1536

    
   
1543

   
1537
    // backwards regions
1544
    // backwards regions
1538
    final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
1545
    final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
1539

    
   
1546

   

    
   
1547
    // sidelined big overlapped regions

    
   
1548
    final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();

    
   
1549

   
1540
    // region split calculator
1550
    // region split calculator
1541
    final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
1551
    final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
1542

    
   
1552

   
1543
    // Histogram of different HTableDescriptors found.  Ideally there is only one!
1553
    // Histogram of different HTableDescriptors found.  Ideally there is only one!
1544
    final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
1554
    final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
[+20] [20] 11 lines
[+20] public class TableInfo {
1556
     * @return descriptor common to all regions.  null if are none or multiple!
1566
     * @return descriptor common to all regions.  null if are none or multiple!
1557
     */
1567
     */
1558
    private HTableDescriptor getHTD() {
1568
    private HTableDescriptor getHTD() {
1559
      if (htds.size() == 1) {
1569
      if (htds.size() == 1) {
1560
        return (HTableDescriptor)htds.toArray()[0];
1570
        return (HTableDescriptor)htds.toArray()[0];

    
   
1571
      } else {

    
   
1572
        LOG.error("None/Multiple table descriptors found for table '"

    
   
1573
          + tableName + "' regions: " + htds);
1561
      }
1574
      }
1562
      return null;
1575
      return null;
1563
    }
1576
    }
1564

    
   
1577

   
1565
    public void addRegionInfo(HbckInfo hir) {
1578
    public void addRegionInfo(HbckInfo hir) {
[+20] [20] 170 lines
[+20] [+] public void handleOverlapGroup(Collection<HbckInfo> overlap)
1736
          return;
1749
          return;
1737
        }
1750
        }
1738

    
   
1751

   
1739
        if (overlap.size() > maxMerge) {
1752
        if (overlap.size() > maxMerge) {
1740
          LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
1753
          LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
1741
              "regions which is greater than " + maxMerge + ", the max " +
1754
            "regions which is greater than " + maxMerge + ", the max number of regions to merge");
1742
              "number of regions to merge.");
1755
          if (sidelineBigOverlaps) {

    
   
1756
            // we only sideline big overlapped groups that exceeds the max number of regions to merge

    
   
1757
            sidelineBigOverlaps(overlap);

    
   
1758
          }
1743
          return;
1759
          return;
1744
        }
1760
        }
1745

    
   
1761

   

    
   
1762
        mergeOverlaps(overlap);

    
   
1763
      }

    
   
1764

   

    
   
1765
      void mergeOverlaps(Collection<HbckInfo> overlap)

    
   
1766
          throws IOException {
1746
        LOG.info("== Merging regions into one region: "
1767
        LOG.info("== Merging regions into one region: "
1747
            + Joiner.on(",").join(overlap));
1768
          + Joiner.on(",").join(overlap));
1748
        // get the min / max range and close all concerned regions
1769
        // get the min / max range and close all concerned regions
1749
        Pair<byte[], byte[]> range = null;
1770
        Pair<byte[], byte[]> range = null;
1750
        for (HbckInfo hi : overlap) {
1771
        for (HbckInfo hi : overlap) {
[+20] [20] 55 lines
[+20] public void handleOverlapGroup(Collection<HbckInfo> overlap)
1806
        }
1827
        }
1807
        if (didFix) {
1828
        if (didFix) {
1808
          fixes++;
1829
          fixes++;
1809
        }
1830
        }
1810
      }
1831
      }
1811
    };

   
1812

    
   
1832

   
1813
    /**
1833
      /**

    
   
1834
       * Sideline some regions in a big overlap group so that it

    
   
1835
       * will have fewer regions, and it is easier to merge them later on.

    
   
1836
       *

    
   
1837
       * @param bigOverlap the overlapped group with regions more than maxMerge

    
   
1838
       * @throws IOException

    
   
1839
       */

    
   
1840
      void sidelineBigOverlaps(

    
   
1841
          Collection<HbckInfo> bigOverlap) throws IOException {

    
   
1842
        int overlapsToSideline = bigOverlap.size() - maxMerge;

    
   
1843
        if (overlapsToSideline > maxOverlapsToSideline) {

    
   
1844
          overlapsToSideline = maxOverlapsToSideline;

    
   
1845
        }

    
   
1846
        List<HbckInfo> regionsToSideline =

    
   
1847
          RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);

    
   
1848
        FileSystem fs = FileSystem.get(conf);

    
   
1849
        for (HbckInfo regionToSideline: regionsToSideline) {

    
   
1850
          try {

    
   
1851
            LOG.info("Closing region: " + regionToSideline);

    
   
1852
            closeRegion(regionToSideline);

    
   
1853
          } catch (InterruptedException ie) {

    
   
1854
            LOG.warn("Was unable to close region " + regionToSideline.getRegionNameAsString()

    
   
1855
              + ".  Interrupted.");

    
   
1856
            throw new IOException(ie);

    
   
1857
          }

    
   
1858

   

    
   
1859
          LOG.info("Offlining region: " + regionToSideline);

    
   
1860
          offline(regionToSideline.getRegionName());

    
   
1861

   

    
   
1862
          LOG.debug("Before sideline big overlapped region: " + regionToSideline.toString());

    
   
1863
          Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline);

    
   
1864
          if (sidelineRegionDir != null) {

    
   
1865
            sidelinedRegions.put(sidelineRegionDir, regionToSideline);

    
   
1866
            LOG.debug("After sidelined big overlapped region: "

    
   
1867
              + regionToSideline.getRegionNameAsString()

    
   
1868
              + " to " + sidelineRegionDir.toString());

    
   
1869
            fixes++;

    
   
1870
          }

    
   
1871
        }

    
   
1872
      }

    
   
1873
    }

    
   
1874

   

    
   
1875
    /**
1814
     * Check the region chain (from META) of this table.  We are looking for
1876
     * Check the region chain (from META) of this table.  We are looking for
1815
     * holes, overlaps, and cycles.
1877
     * holes, overlaps, and cycles.
1816
     * @return false if there are errors
1878
     * @return false if there are errors
1817
     * @throws IOException
1879
     * @throws IOException
1818
     */
1880
     */
[+20] [20] 73 lines
[+20] [+] public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
1892
        handler.handleOverlapGroup(overlap);
1954
        handler.handleOverlapGroup(overlap);
1893
      }
1955
      }
1894

    
   
1956

   
1895
      if (details) {
1957
      if (details) {
1896
        // do full region split map dump
1958
        // do full region split map dump
1897
        System.out.println("---- Table '"  +  this.tableName 
1959
        System.out.println("---- Table '"  +  this.tableName
1898
            + "': region split map");
1960
            + "': region split map");
1899
        dump(splits, regions);
1961
        dump(splits, regions);
1900
        System.out.println("---- Table '"  +  this.tableName 
1962
        System.out.println("---- Table '"  +  this.tableName
1901
            + "': overlap groups");
1963
            + "': overlap groups");
1902
        dumpOverlapProblems(overlapGroups);
1964
        dumpOverlapProblems(overlapGroups);
1903
        System.out.println("There are " + overlapGroups.keySet().size()
1965
        System.out.println("There are " + overlapGroups.keySet().size()
1904
            + " overlap groups with " + overlapGroups.size()
1966
            + " overlap groups with " + overlapGroups.size()
1905
            + " overlapping regions");
1967
            + " overlapping regions");
1906
      }
1968
      }

    
   
1969
      if (!sidelinedRegions.isEmpty()) {

    
   
1970
        LOG.warn("Sidelined big overlapped regions, please bulk load them!");

    
   
1971
        System.out.println("---- Table '"  +  this.tableName

    
   
1972
            + "': sidelined big overlapped regions");

    
   
1973
        dumpSidelinedRegions(sidelinedRegions);

    
   
1974
      }
1907
      return errors.getErrorList().size() == originalErrorsCount;
1975
      return errors.getErrorList().size() == originalErrorsCount;
1908
    }
1976
    }
1909

    
   
1977

   
1910
    /**
1978
    /**
1911
     * This dumps data in a visually reasonable way for visual debugging
1979
     * This dumps data in a visually reasonable way for visual debugging
[+20] [20] 25 lines
[+20] [+] public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
1937
      }
2005
      }
1938
      System.out.println("----");
2006
      System.out.println("----");
1939
    }
2007
    }
1940
  }
2008
  }
1941

    
   
2009

   

    
   
2010
  public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {

    
   
2011
    for (Path k : regions.keySet()) {

    
   
2012
      System.out.println("To be bulk loaded sidelined region dir: "

    
   
2013
        + k.toString());

    
   
2014
    }

    
   
2015
  }

    
   
2016

   
1942
  public Multimap<byte[], HbckInfo> getOverlapGroups(
2017
  public Multimap<byte[], HbckInfo> getOverlapGroups(
1943
      String table) {
2018
      String table) {
1944
    TableInfo ti = tablesInfo.get(table);
2019
    TableInfo ti = tablesInfo.get(table);
1945
    return ti.overlapGroups;
2020
    return ti.overlapGroups;
1946
  }
2021
  }
[+20] [20] 789 lines
[+20] [+] public void setFixHdfsOrphans(boolean shouldFix) {
2736

    
   
2811

   
2737
  boolean shouldFixHdfsOrphans() {
2812
  boolean shouldFixHdfsOrphans() {
2738
    return fixHdfsOrphans;
2813
    return fixHdfsOrphans;
2739
  }
2814
  }
2740

    
   
2815

   

    
   
2816
  public void setSidelineBigOverlaps(boolean sbo) {

    
   
2817
    this.sidelineBigOverlaps = sbo;

    
   
2818
  }

    
   
2819

   

    
   
2820
  public boolean shouldSidelineBigOverlaps() {

    
   
2821
    return sidelineBigOverlaps;

    
   
2822
  }

    
   
2823

   
2741
  /**
2824
  /**
2742
   * @param mm maximum number of regions to merge into a single region.
2825
   * @param mm maximun number of regions to merge into a single region.
2743
   */
2826
   */
2744
  public void setMaxMerge(int mm) {
2827
  public void setMaxMerge(int mm) {
2745
    this.maxMerge = mm;
2828
    this.maxMerge = mm;
2746
  }
2829
  }
2747

    
   
2830

   
2748
  public int getMaxMerge() {
2831
  public int getMaxMerge() {
2749
    return maxMerge;
2832
    return maxMerge;
2750
  }
2833
  }
2751

    
   
2834

   

    
   
2835
  public void setMaxOverlapsToSideline(int mo) {

    
   
2836
    this.maxOverlapsToSideline = mo;

    
   
2837
  }

    
   
2838

   

    
   
2839
  public int getMaxOverlapsToSideline() {

    
   
2840
    return maxOverlapsToSideline;

    
   
2841
  }

    
   
2842

   
2752
  /**
2843
  /**
2753
   * Only fix tables specified by the list
2844
   * Only fix tables specified by the list
2754
   */
2845
   */
2755
  boolean shouldFixTable(byte[] table) {
2846
  boolean shouldFixTable(byte[] table) {
2756
    if (tablesToFix.size() == 0) {
2847
    if (tablesToFix.size() == 0) {
[+20] [20] 38 lines
[+20] [+] protected static void printUsageAndExit() {
2795
    System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
2886
    System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
2796
    System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
2887
    System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
2797
    System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
2888
    System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
2798
    System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
2889
    System.err.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
2799
    System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
2890
    System.err.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");

    
   
2891
    System.err.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");

    
   
2892
    System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
2800
    System.err.println("");
2893
    System.err.println("");
2801
    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps");
2894
    System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps -sidelineBigOverlaps");
2802
    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
2895
    System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
2803

    
   
2896

   
2804
    Runtime.getRuntime().exit(-2);
2897
    Runtime.getRuntime().exit(-2);
2805
  }
2898
  }
2806

    
   
2899

   
[+20] [20] 52 lines
[+20] [+] public static void main(String[] args) throws Exception {
2859
        fsck.setFixHdfsHoles(true);
2952
        fsck.setFixHdfsHoles(true);
2860
      } else if (cmd.equals("-fixHdfsOrphans")) {
2953
      } else if (cmd.equals("-fixHdfsOrphans")) {
2861
        fsck.setFixHdfsOrphans(true);
2954
        fsck.setFixHdfsOrphans(true);
2862
      } else if (cmd.equals("-fixHdfsOverlaps")) {
2955
      } else if (cmd.equals("-fixHdfsOverlaps")) {
2863
        fsck.setFixHdfsOverlaps(true);
2956
        fsck.setFixHdfsOverlaps(true);

    
   
2957
      } else if (cmd.equals("-sidelineBigOverlaps")) {

    
   
2958
        fsck.setSidelineBigOverlaps(true);
2864
      } else if (cmd.equals("-repair")) {
2959
      } else if (cmd.equals("-repair")) {
2865
        // this attempts to merge overlapping hdfs regions, needs testing
2960
        // this attempts to merge overlapping hdfs regions, needs testing
2866
        // under load
2961
        // under load
2867
        fsck.setFixHdfsHoles(true);
2962
        fsck.setFixHdfsHoles(true);
2868
        fsck.setFixHdfsOrphans(true);
2963
        fsck.setFixHdfsOrphans(true);
2869
        fsck.setFixMeta(true);
2964
        fsck.setFixMeta(true);
2870
        fsck.setFixAssignments(true);
2965
        fsck.setFixAssignments(true);
2871
        fsck.setFixHdfsOverlaps(true);
2966
        fsck.setFixHdfsOverlaps(true);

    
   
2967
        fsck.setSidelineBigOverlaps(true);
2872
      } else if (cmd.equals("-repairHoles")) {
2968
      } else if (cmd.equals("-repairHoles")) {
2873
        // this will make all missing hdfs regions available but may lose data
2969
        // this will make all missing hdfs regions available but may lose data
2874
        fsck.setFixHdfsHoles(true);
2970
        fsck.setFixHdfsHoles(true);
2875
        fsck.setFixHdfsOrphans(false);
2971
        fsck.setFixHdfsOrphans(false);
2876
        fsck.setFixMeta(true);
2972
        fsck.setFixMeta(true);
2877
        fsck.setFixAssignments(true);
2973
        fsck.setFixAssignments(true);
2878
        fsck.setFixHdfsOverlaps(false);
2974
        fsck.setFixHdfsOverlaps(false);

    
   
2975
        fsck.setSidelineBigOverlaps(false);

    
   
2976
      } else if (cmd.equals("-maxOverlapsToSideline")) {

    
   
2977
        if (i == args.length - 1) {

    
   
2978
          System.err.println("HBaseFsck: -maxOverlapsToSideline needs a value.");

    
   
2979
          printUsageAndExit();

    
   
2980
        }

    
   
2981
        try {

    
   
2982
          int maxOverlapsToSideline = Integer.parseInt(args[i+1]);

    
   
2983
          fsck.setMaxOverlapsToSideline(maxOverlapsToSideline);

    
   
2984
        } catch (NumberFormatException e) {

    
   
2985
          System.err.println("-maxOverlapsToSideline needs a numeric value.");

    
   
2986
          printUsageAndExit();

    
   
2987
        }

    
   
2988
        i++;
2879
      } else if (cmd.equals("-maxMerge")) {
2989
      } else if (cmd.equals("-maxMerge")) {
2880
        if (i == args.length - 1) {
2990
        if (i == args.length - 1) {
2881
          System.err.println("-maxMerge needs a numeric value argument.");
2991
          System.err.println("-maxMerge needs a numeric value argument.");
2882
          printUsageAndExit();
2992
          printUsageAndExit();
2883
        }
2993
        }
[+20] [20] 78 lines
src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
Revision 17678dd New Change
 
src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
Revision ac3b225 New Change
 
  1. src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java: Loading...
  3. src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java: Loading...