Review Board 1.7.22


HBase-4348 Add metrics for regions in transition

Review Request #4402 - Created March 19, 2012 and updated

Himanshu Vashishtha
trunk
HBase-4348
Reviewers
hbase
hbase-git
This patch is for adding Region in transition metrics to the HMaster metrics system. It also adds these metrics in the master ui, in the Region in transition section. I have attached the proposed new format in the jira 4348.
Ran on a 5 node cluster and kill region servers randomly to observe the changes in the RIT metrics as emitted out by the Master's mxbean;

mvn test passes without any failure.
src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
Revision 0dc0691 New Change
[20] 18 lines
[+20]
19
</%doc>
19
</%doc>
20
<%import>
20
<%import>
21
org.apache.hadoop.hbase.HRegionInfo;
21
org.apache.hadoop.hbase.HRegionInfo;
22
org.apache.hadoop.hbase.master.AssignmentManager;
22
org.apache.hadoop.hbase.master.AssignmentManager;
23
org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
23
org.apache.hadoop.hbase.master.AssignmentManager.RegionState;

    
   
24
org.apache.hadoop.conf.Configuration;

    
   
25
org.apache.hadoop.hbase.HBaseConfiguration;
24
java.util.Iterator;
26
java.util.Iterator;
25
java.util.Map;
27
java.util.Map;
26
</%import>
28
</%import>
27
<%args>
29
<%args>
28
AssignmentManager assignmentManager;
30
AssignmentManager assignmentManager;
29
int limit = 100;
31
int limit = 100;
30
</%args>
32
</%args>
31
<%java>
33
<%java>
32
Map<String, RegionState> rit = assignmentManager.getRegionsInTransition();
34
Map<String, RegionState> rit = assignmentManager.getRegionsInTransition();

    
   
35
// process the map to find region in transition details

    
   
36
Configuration conf = HBaseConfiguration.create();

    
   
37
int ritThreshold = conf.getInt("rit.metrics.threshold.time", 60000);

    
   
38
int numOfRITOverThreshold = 0;

    
   
39
long maxRITTime = Long.MIN_VALUE;

    
   
40
long currentTime = System.currentTimeMillis();

    
   
41
String regionIDForOldestRIT = ""; // avoiding null

    
   
42
for (Map.Entry<String, RegionState> e : rit.entrySet()) {

    
   
43
  long ritTime = currentTime - e.getValue().getStamp();

    
   
44
  if(ritTime > ritThreshold) {

    
   
45
     numOfRITOverThreshold++;

    
   
46
   }

    
   
47
   if(maxRITTime < ritTime) {

    
   
48
     maxRITTime = ritTime;

    
   
49
     regionIDForOldestRIT = e.getKey();

    
   
50
   }

    
   
51
}
33

    
   
52

   
34
int toRemove = rit.size() - limit;
53
int toRemove = rit.size() - limit;
35
int removed = 0;
54
int removed = 0;
36
if (toRemove > 0) {
55
if (toRemove > 0) {
37
  // getRegionsInTransition returned a copy, so we can mutate it
56
  // getRegionsInTransition returned a copy, so we can mutate it
38
  for (Iterator<Map.Entry<String, RegionState>> it = rit.entrySet().iterator();
57
  for (Iterator<Map.Entry<String, RegionState>> it = rit.entrySet().iterator();
39
       it.hasNext() && toRemove > 0;
58
       it.hasNext() && toRemove > 0;
40
       ) {
59
       ) {
41
    Map.Entry<String, RegionState> e = it.next();
60
    Map.Entry<String, RegionState> e = it.next();
42
    if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals(
61
    if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals(
43
          e.getKey()) ||
62
          e.getKey()) ||
44
        HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals(
63
        HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals(
45
          e.getKey())) {
64
          e.getKey()) || regionIDForOldestRIT.equals(e.getKey())) {
46
      // don't remove the meta regions, they're too interesting!
65
      // don't remove the meta & the oldest rit regions, they're too interesting!
47
      continue;
66
      continue;
48
    } 
67
    } 
49
    it.remove();
68
    it.remove();
50
    toRemove--;
69
    toRemove--;
51
    removed++;
70
    removed++;
[+20] [20] 5 lines
[+20]
57
<h2>Regions in Transition</h2>
76
<h2>Regions in Transition</h2>
58
<%if rit.isEmpty() %>
77
<%if rit.isEmpty() %>
59
No regions in transition.
78
No regions in transition.
60
<%else>
79
<%else>
61
	<table>
80
        <table>
62
		<tr><th>Region</th><th>State</th></tr>
81
                <tr><th>Region</th><th>State</th><th>RIT time (ms)</th></tr>
63
		<%for Map.Entry<String, RegionState> entry : rit.entrySet() %>
82
                <%for Map.Entry<String, RegionState> entry : rit.entrySet() %>
64
		<tr><td><% entry.getKey() %></td><td><% entry.getValue().toDescriptiveString() %></td>
83
                <%if regionIDForOldestRIT.equals(entry.getKey()) %>

    
   
84
                        <tr BGCOLOR="#FE2E2E" >

    
   
85
                <%else>

    
   
86
                        <tr>

    
   
87
                </%if>

    
   
88
                <td><% entry.getKey() %></td><td><% entry.getValue().toDescriptiveString() %></td>

    
   
89
		<td><% (currentTime - entry.getValue().getStamp()) %> </td></tr>
65
		</%for>
90
                </%for>

    
   
91
                <tr BGCOLOR="#D7DF01"> <td>Total number of Regions in Transition for more than <% ritThreshold %> milliseconds</td><td> <% numOfRITOverThreshold %></td><td></td>

    
   
92
                </tr>

    
   
93
		<tr> <td> Total number of Regions in Transition</td><td><% rit.size() %> </td><td></td>
66
	</table>
94
        </table>
67
	<%if removed > 0 %>
95
        <%if removed > 0 %>
68
	(<% removed %> more regions in transition not shown) 
96
        (<% removed %> more regions in transition not shown)
69
	</%if>
97
        </%if>
70
</%if>
98
</%if>

    
   
99

   
src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
Revision ae468ca New Change
 
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Revision c4b4d30 New Change
 
src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java
Revision 83abc52 New Change
 
src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
Revision d68ce33 New Change
 
  1. src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon: Loading...
  2. src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/master/HMaster.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java: Loading...
  5. src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java: Loading...