Review Board 1.7.22


HBase-4348 Add metrics for regions in transition

Review Request #4402 - Created March 19, 2012 and updated

Himanshu Vashishtha
trunk
HBase-4348
Reviewers
hbase
hbase-git
This patch is for adding Region in transition metrics to the HMaster metrics system. It also adds these metrics in the master ui, in the Region in transition section. I have attached the proposed new format in the jira 4348.
Ran on a 5 node cluster and kill region servers randomly to observe the changes in the RIT metrics as emitted out by the Master's mxbean;

mvn test passes without any failure.
src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
Revision 0dc0691 New Change
[20] 18 lines
[+20]
19
</%doc>
19
</%doc>
20
<%import>
20
<%import>
21
org.apache.hadoop.hbase.HRegionInfo;
21
org.apache.hadoop.hbase.HRegionInfo;
22
org.apache.hadoop.hbase.master.AssignmentManager;
22
org.apache.hadoop.hbase.master.AssignmentManager;
23
org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
23
org.apache.hadoop.hbase.master.AssignmentManager.RegionState;

    
   
24
org.apache.hadoop.conf.Configuration;

    
   
25
org.apache.hadoop.hbase.HBaseConfiguration;

    
   
26
org.apache.hadoop.hbase.HConstants;
24
java.util.Iterator;
27
java.util.Iterator;
25
java.util.Map;
28
java.util.Map;
26
</%import>
29
</%import>
27
<%args>
30
<%args>
28
AssignmentManager assignmentManager;
31
AssignmentManager assignmentManager;
29
int limit = 100;
32
int limit = 100;
30
</%args>
33
</%args>
31
<%java>
34
<%java>
32
Map<String, RegionState> rit = assignmentManager.getRegionsInTransition();
35
Map<String, RegionState> rit = assignmentManager.getRegionsInTransition();

    
   
36
// process the map to find region in transition details

    
   
37
Configuration conf = HBaseConfiguration.create();

    
   
38
int ritThreshold = conf.getInt(HConstants.METRICS_RIT_THRESHOLD_TIME, 60000);

    
   
39
int numOfRITOverThreshold = 0;

    
   
40
long maxRITTime = Long.MIN_VALUE;

    
   
41
long currentTime = System.currentTimeMillis();

    
   
42
String regionIDForOldestRIT = ""; // avoiding null

    
   
43
for (Map.Entry<String, RegionState> e : rit.entrySet()) {

    
   
44
  long ritTime = currentTime - e.getValue().getStamp();

    
   
45
  if(ritTime > ritThreshold) {

    
   
46
     numOfRITOverThreshold++;

    
   
47
   }

    
   
48
   if(maxRITTime < ritTime) {

    
   
49
     maxRITTime = ritTime;

    
   
50
     regionIDForOldestRIT = e.getKey();

    
   
51
   }

    
   
52
}
33

    
   
53

   
34
int toRemove = rit.size() - limit;
54
int toRemove = rit.size() - limit;
35
int removed = 0;
55
int removed = 0;
36
if (toRemove > 0) {
56
if (toRemove > 0) {
37
  // getRegionsInTransition returned a copy, so we can mutate it
57
  // getRegionsInTransition returned a copy, so we can mutate it
38
  for (Iterator<Map.Entry<String, RegionState>> it = rit.entrySet().iterator();
58
  for (Iterator<Map.Entry<String, RegionState>> it = rit.entrySet().iterator();
39
       it.hasNext() && toRemove > 0;
59
       it.hasNext() && toRemove > 0;
40
       ) {
60
       ) {
41
    Map.Entry<String, RegionState> e = it.next();
61
    Map.Entry<String, RegionState> e = it.next();
42
    if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals(
62
    if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals(
43
          e.getKey()) ||
63
          e.getKey()) ||
44
        HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals(
64
        HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals(
45
          e.getKey())) {
65
          e.getKey()) || regionIDForOldestRIT.equals(e.getKey())) {
46
      // don't remove the meta regions, they're too interesting!
66
      // don't remove the meta & the oldest rit regions, they're too interesting!
47
      continue;
67
      continue;
48
    } 
68
    } 
49
    it.remove();
69
    it.remove();
50
    toRemove--;
70
    toRemove--;
51
    removed++;
71
    removed++;
[+20] [20] 5 lines
[+20]
57
<h2>Regions in Transition</h2>
77
<h2>Regions in Transition</h2>
58
<%if rit.isEmpty() %>
78
<%if rit.isEmpty() %>
59
No regions in transition.
79
No regions in transition.
60
<%else>
80
<%else>
61
	<table>
81
        <table>
62
		<tr><th>Region</th><th>State</th></tr>
82
                <tr><th>Region</th><th>State</th><th>RIT time (ms)</th></tr>
63
		<%for Map.Entry<String, RegionState> entry : rit.entrySet() %>
83
                <%for Map.Entry<String, RegionState> entry : rit.entrySet() %>
64
		<tr><td><% entry.getKey() %></td><td><% entry.getValue().toDescriptiveString() %></td>
84
                <%if regionIDForOldestRIT.equals(entry.getKey()) %>

    
   
85
                        <tr BGCOLOR="#FE2E2E" >

    
   
86
                <%else>

    
   
87
                        <tr>

    
   
88
                </%if>

    
   
89
                <td><% entry.getKey() %></td><td><% entry.getValue().toDescriptiveString() %></td>

    
   
90
		<td><% (currentTime - entry.getValue().getStamp()) %> </td></tr>
65
		</%for>
91
                </%for>

    
   
92
                <tr BGCOLOR="#D7DF01"> <td>Total number of Regions in Transition for more than <% ritThreshold %> milliseconds</td><td> <% numOfRITOverThreshold %></td><td></td>

    
   
93
                </tr>

    
   
94
		<tr> <td> Total number of Regions in Transition</td><td><% rit.size() %> </td><td></td>
66
	</table>
95
        </table>
67
	<%if removed > 0 %>
96
        <%if removed > 0 %>
68
	(<% removed %> more regions in transition not shown) 
97
        (<% removed %> more regions in transition not shown)
69
	</%if>
98
        </%if>
70
</%if>
99
</%if>

    
   
100

   
src/main/java/org/apache/hadoop/hbase/HConstants.java
Revision 19be4de New Change
 
src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
Revision ae468ca New Change
 
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Revision c4b4d30 New Change
 
src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
Revision 83abc52 New Change
 
src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
Revision d68ce33 New Change
 
  1. src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon: Loading...
  2. src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/master/HMaster.java: Loading...
  5. src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java: Loading...
  6. src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java: Loading...