Review Board 1.7.22


HBASE-1502 Remove need for heartbeats in HBase

Review Request #647 - Created April 22, 2011 and updated

Michael Stack
trunk
hbase-1502
Reviewers
hbase
hbase-git
This patch does not completely remove heartbeats.  It unburdens the heartbeat of control messages; now heartbeat is used to
send the master load only (At most recent hackathon we had rough agreement that we'd keep heartbeat to carry load)... if we miss some, no biggie.   

RPC version changed on HMasterRegionInfo since the regionServerStartup and regionServerReport arguments have changed.
We pass a String now instead of HServerAddress so this should help with our DNS issues where the two sides disagree.

Removed HMsg.

HServerAddress as been sort_of_deprecated.  Its in our API so can't remove it easily (its embedded inside HRegionLocation).
Otherwise, we don't use it internally anymore.

HServerInfo is deprecated.  Server meta data is now available in new class ServerName and load lives apart from HSI now.

Fixed up regionserver and master startup so they now look the same.

New tests

Cruft cleanup.
Most tests pass.  I have yet to run it on a cluster.  Doing that now.
src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
Revision 26a8bef New Change
[20] 23 lines
[+20]
24
import java.io.DataOutput;
24
import java.io.DataOutput;
25
import java.io.IOException;
25
import java.io.IOException;
26
import java.util.ArrayList;
26
import java.util.ArrayList;
27
import java.util.Collection;
27
import java.util.Collection;
28
import java.util.Collections;
28
import java.util.Collections;

    
   
29
import java.util.HashMap;
29
import java.util.Map;
30
import java.util.Map;
30
import java.util.TreeMap;
31
import java.util.TreeMap;
31

    
   
32

   
32
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
33
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
33
import org.apache.hadoop.io.VersionedWritable;
34
import org.apache.hadoop.io.VersionedWritable;
[+20] [20] 19 lines
[+20] [+] public class ClusterStatus extends VersionedWritable {
53
   * Version for object serialization.  Incremented for changes in serialized
54
   * Version for object serialization.  Incremented for changes in serialized
54
   * representation.
55
   * representation.
55
   * <dl>
56
   * <dl>
56
   *   <dt>0</dt> <dd>initial version</dd>
57
   *   <dt>0</dt> <dd>initial version</dd>
57
   *   <dt>1</dt> <dd>added cluster ID</dd>
58
   *   <dt>1</dt> <dd>added cluster ID</dd>

    
   
59
   *   <dt>2</dt> <dd>Added Map of ServerName to ServerLoad</dd>
58
   * </dl>
60
   * </dl>
59
   */
61
   */
60
  private static final byte VERSION = 1;
62
  private static final byte VERSION = 2;
61

    
   
63

   
62
  private String hbaseVersion;
64
  private String hbaseVersion;
63
  private Collection<HServerInfo> liveServerInfo;
65
  private Map<ServerName, HServerLoad> liveServers;
64
  private Collection<String> deadServers;
66
  private Collection<ServerName> deadServers;
65
  private Map<String, RegionState> intransition;
67
  private Map<String, RegionState> intransition;
66
  private String clusterId;
68
  private String clusterId;
67

    
   
69

   
68
  /**
70
  /**
69
   * Constructor, for Writable
71
   * Constructor, for Writable
70
   */
72
   */
71
  public ClusterStatus() {
73
  public ClusterStatus() {
72
    super();
74
    super();
73
  }
75
  }
74

    
   
76

   

    
   
77
  public ClusterStatus(final String hbaseVersion, final String clusterid,

    
   
78
      final Map<ServerName, HServerLoad> servers,

    
   
79
      final Collection<ServerName> deadServers, final Map<String, RegionState> rit) {

    
   
80
    this.hbaseVersion = hbaseVersion;

    
   
81
    this.liveServers = servers;

    
   
82
    this.deadServers = deadServers;

    
   
83
    this.intransition = rit;

    
   
84
    this.clusterId = clusterid;

    
   
85
  }

    
   
86

   
75
  /**
87
  /**
76
   * @return the names of region servers on the dead list
88
   * @return the names of region servers on the dead list
77
   */
89
   */
78
  public Collection<String> getDeadServerNames() {
90
  public Collection<ServerName> getDeadServerNames() {
79
    return Collections.unmodifiableCollection(deadServers);
91
    return Collections.unmodifiableCollection(deadServers);
80
  }
92
  }
81

    
   
93

   
82
  /**
94
  /**
83
   * @return the number of region servers in the cluster
95
   * @return the number of region servers in the cluster
84
   */
96
   */
85
  public int getServers() {
97
  public int getServersSize() {
86
    return liveServerInfo.size();
98
    return liveServers.size();
87
  }
99
  }
88

    
   
100

   
89
  /**
101
  /**
90
   * @return the number of dead region servers in the cluster
102
   * @return the number of dead region servers in the cluster
91
   */
103
   */
[+20] [20] 4 lines
[+20] [+] public int getDeadServers() {
96
  /**
108
  /**
97
   * @return the average cluster load
109
   * @return the average cluster load
98
   */
110
   */
99
  public double getAverageLoad() {
111
  public double getAverageLoad() {
100
    int load = 0;
112
    int load = 0;
101
    for (HServerInfo server: liveServerInfo) {
113
    for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
102
      load += server.getLoad().getLoad();
114
      load += e.getValue().getLoad();
103
    }
115
    }
104
    return (double)load / (double)liveServerInfo.size();
116
    return (double)load / (double)getServersSize();
105
  }
117
  }
106

    
   
118

   
107
  /**
119
  /**
108
   * @return the number of regions deployed on the cluster
120
   * @return the number of regions deployed on the cluster
109
   */
121
   */
110
  public int getRegionsCount() {
122
  public int getRegionsCount() {
111
    int count = 0;
123
    int count = 0;
112
    for (HServerInfo server: liveServerInfo) {
124
    for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
113
      count += server.getLoad().getNumberOfRegions();
125
      count += e.getValue().getNumberOfRegions();
114
    }
126
    }
115
    return count;
127
    return count;
116
  }
128
  }
117

    
   
129

   
118
  /**
130
  /**
119
   * @return the number of requests since last report
131
   * @return the number of requests since last report
120
   */
132
   */
121
  public int getRequestsCount() {
133
  public int getRequestsCount() {
122
    int count = 0;
134
    int count = 0;
123
    for (HServerInfo server: liveServerInfo) {
135
    for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
124
      count += server.getLoad().getNumberOfRequests();
136
      count += e.getValue().getNumberOfRequests();
125
    }
137
    }
126
    return count;
138
    return count;
127
  }
139
  }
128

    
   
140

   
129
  /**
141
  /**
130
   * @return the HBase version string as reported by the HMaster
142
   * @return the HBase version string as reported by the HMaster
131
   */
143
   */
132
  public String getHBaseVersion() {
144
  public String getHBaseVersion() {
133
    return hbaseVersion;
145
    return hbaseVersion;
134
  }
146
  }
135

    
   
147

   
136
  /**
148
  /**
137
   * @param version the HBase version string

   
138
   */

   
139
  public void setHBaseVersion(String version) {

   
140
    hbaseVersion = version;

   
141
  }

   
142

    
   

   
143
  /**

   
144
   * @see java.lang.Object#equals(java.lang.Object)
149
   * @see java.lang.Object#equals(java.lang.Object)
145
   */
150
   */
146
  public boolean equals(Object o) {
151
  public boolean equals(Object o) {
147
    if (this == o) {
152
    if (this == o) {
148
      return true;
153
      return true;
149
    }
154
    }
150
    if (!(o instanceof ClusterStatus)) {
155
    if (!(o instanceof ClusterStatus)) {
151
      return false;
156
      return false;
152
    }
157
    }
153
    return (getVersion() == ((ClusterStatus)o).getVersion()) &&
158
    return (getVersion() == ((ClusterStatus)o).getVersion()) &&
154
      getHBaseVersion().equals(((ClusterStatus)o).getHBaseVersion()) &&
159
      getHBaseVersion().equals(((ClusterStatus)o).getHBaseVersion()) &&
155
      liveServerInfo.equals(((ClusterStatus)o).liveServerInfo) &&
160
      this.liveServers.equals(((ClusterStatus)o).liveServers) &&
156
      deadServers.equals(((ClusterStatus)o).deadServers);
161
      deadServers.equals(((ClusterStatus)o).deadServers);
157
  }
162
  }
158

    
   
163

   
159
  /**
164
  /**
160
   * @see java.lang.Object#hashCode()
165
   * @see java.lang.Object#hashCode()
161
   */
166
   */
162
  public int hashCode() {
167
  public int hashCode() {
163
    return VERSION + hbaseVersion.hashCode() + liveServerInfo.hashCode() +
168
    return VERSION + hbaseVersion.hashCode() + this.liveServers.hashCode() +
164
      deadServers.hashCode();
169
      deadServers.hashCode();
165
  }
170
  }
166

    
   
171

   
167
  /** @return the object version number */
172
  /** @return the object version number */
168
  public byte getVersion() {
173
  public byte getVersion() {
[+20] [20] 4 lines
[+20] public boolean equals(Object o) {
173
  // Getters
178
  // Getters
174
  //
179
  //
175

    
   
180

   
176
  /**
181
  /**
177
   * Returns detailed region server information: A list of
182
   * Returns detailed region server information: A list of
178
   * {@link HServerInfo}, containing server load and resource usage
183
   * {@link ServerName}.
179
   * statistics as {@link HServerLoad}, containing per-region

   
180
   * statistics as {@link HServerLoad.RegionLoad}.

   
181
   * @return region server information
184
   * @return region server information

    
   
185
   * @deprecated Use {@link #getServers()}
182
   */
186
   */
183
  public Collection<HServerInfo> getServerInfo() {
187
  public Collection<ServerName> getServerInfo() {
184
    return Collections.unmodifiableCollection(liveServerInfo);
188
    return getServers();
185
  }
189
  }
186

    
   
190

   
187
  //
191
  public Collection<ServerName> getServers() {
188
  // Setters
192
    return Collections.unmodifiableCollection(this.liveServers.keySet());
189
  //

   
190

    
   

   
191
  public void setServerInfo(Collection<HServerInfo> serverInfo) {

   
192
    this.liveServerInfo = serverInfo;

   
193
  }
193
  }
194

    
   
194

   
195
  public void setDeadServers(Collection<String> deadServers) {
195
  /**
196
    this.deadServers = deadServers;
196
   * @param sn

    
   
197
   * @return Server's load or null if not found.

    
   
198
   */

    
   
199
  public HServerLoad getLoad(final ServerName sn) {

    
   
200
    return this.liveServers.get(sn);
197
  }
201
  }
198

    
   
202

   
199
  public Map<String, RegionState> getRegionsInTransition() {
203
  public Map<String, RegionState> getRegionsInTransition() {
200
    return this.intransition;
204
    return this.intransition;
201
  }
205
  }
202

    
   
206

   
203
  public void setRegionsInTransition(final Map<String, RegionState> m) {

   
204
    this.intransition = m;

   
205
  }

   
206

    
   

   
207
  public String getClusterId() {
207
  public String getClusterId() {
208
    return clusterId;
208
    return clusterId;
209
  }
209
  }
210

    
   
210

   
211
  public void setClusterId(String id) {

   
212
    this.clusterId = id;

   
213
  }

   
214

    
   

   
215
  //
211
  //
216
  // Writable
212
  // Writable
217
  //
213
  //
218

    
   
214

   
219
  public void write(DataOutput out) throws IOException {
215
  public void write(DataOutput out) throws IOException {
220
    super.write(out);
216
    super.write(out);
221
    out.writeUTF(hbaseVersion);
217
    out.writeUTF(hbaseVersion);
222
    out.writeInt(liveServerInfo.size());
218
    out.writeInt(getServersSize());
223
    for (HServerInfo server: liveServerInfo) {
219
    for (Map.Entry<ServerName, HServerLoad> e: this.liveServers.entrySet()) {
224
      server.write(out);
220
      out.writeUTF(e.getKey().toString());

    
   
221
      e.getValue().write(out);
225
    }
222
    }
226
    out.writeInt(deadServers.size());
223
    out.writeInt(deadServers.size());
227
    for (String server: deadServers) {
224
    for (ServerName server: deadServers) {
228
      out.writeUTF(server);
225
      out.writeUTF(server.toString());
229
    }
226
    }
230
    out.writeInt(this.intransition.size());
227
    out.writeInt(this.intransition.size());
231
    for (Map.Entry<String, RegionState> e: this.intransition.entrySet()) {
228
    for (Map.Entry<String, RegionState> e: this.intransition.entrySet()) {
232
      out.writeUTF(e.getKey());
229
      out.writeUTF(e.getKey());
233
      e.getValue().write(out);
230
      e.getValue().write(out);
234
    }
231
    }
235
    out.writeUTF(clusterId);
232
    out.writeUTF(clusterId);
236
  }
233
  }
237

    
   
234

   
238
  public void readFields(DataInput in) throws IOException {
235
  public void readFields(DataInput in) throws IOException {
239
    super.readFields(in);
236
    super.readFields(in);
240
    hbaseVersion = in.readUTF();
237
    hbaseVersion = in.readUTF();
241
    int count = in.readInt();
238
    int count = in.readInt();
242
    liveServerInfo = new ArrayList<HServerInfo>(count);
239
    this.liveServers = new HashMap<ServerName, HServerLoad>(count);
243
    for (int i = 0; i < count; i++) {
240
    for (int i = 0; i < count; i++) {
244
      HServerInfo info = new HServerInfo();
241
      String str = in.readUTF();
245
      info.readFields(in);
242
      HServerLoad hsl = new HServerLoad();
246
      liveServerInfo.add(info);
243
      hsl.readFields(in);

    
   
244
      this.liveServers.put(new ServerName(str), hsl);
247
    }
245
    }
248
    count = in.readInt();
246
    count = in.readInt();
249
    deadServers = new ArrayList<String>(count);
247
    deadServers = new ArrayList<ServerName>(count);
250
    for (int i = 0; i < count; i++) {
248
    for (int i = 0; i < count; i++) {
251
      deadServers.add(in.readUTF());
249
      deadServers.add(new ServerName(in.readUTF()));
252
    }
250
    }
253
    count = in.readInt();
251
    count = in.readInt();
254
    this.intransition = new TreeMap<String, RegionState>();
252
    this.intransition = new TreeMap<String, RegionState>();
255
    for (int i = 0; i < count; i++) {
253
    for (int i = 0; i < count; i++) {
256
      String key = in.readUTF();
254
      String key = in.readUTF();
257
      RegionState regionState = new RegionState();
255
      RegionState regionState = new RegionState();
258
      regionState.readFields(in);
256
      regionState.readFields(in);
259
      this.intransition.put(key, regionState);
257
      this.intransition.put(key, regionState);
260
    }
258
    }
261
    this.clusterId = in.readUTF();
259
    this.clusterId = in.readUTF();
262
  }
260
  }
263
}
261
}
src/main/java/org/apache/hadoop/hbase/HConstants.java
Revision 5701639 New Change
 
src/main/java/org/apache/hadoop/hbase/HMsg.java
Revision 87beb00 New Change
 
src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
Revision bd353b8 New Change
 
src/main/java/org/apache/hadoop/hbase/HServerAddress.java
Revision 7f8a472 New Change
 
src/main/java/org/apache/hadoop/hbase/HServerInfo.java
Revision 0b5bd94 New Change
 
src/main/java/org/apache/hadoop/hbase/HServerLoad.java
Revision 2372053 New Change
 
src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
Revision 0d696ab New Change
 
src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java
Revision 1da9742 New Change
 
src/main/java/org/apache/hadoop/hbase/Server.java
Revision df396fa New Change
 
src/main/java/org/apache/hadoop/hbase/ServerName.java
New File
 
src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java
Revision d7a1e67 New Change
 
src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
Revision be31179 New Change
 
src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Revision c2ee031 New Change
 
src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
Revision 6e22cf5 New Change
 
src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java
Revision aee64c5 New Change
 
src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
Revision afb666a New Change
 
src/main/java/org/apache/hadoop/hbase/client/HConnection.java
Revision 2bb4725 New Change
 
src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Revision c348f7a New Change
 
src/main/java/org/apache/hadoop/hbase/client/HTable.java
Revision edacf56 New Change
 
  1. src/main/java/org/apache/hadoop/hbase/ClusterStatus.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/HMsg.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/HRegionLocation.java: Loading...
  5. src/main/java/org/apache/hadoop/hbase/HServerAddress.java: Loading...
  6. src/main/java/org/apache/hadoop/hbase/HServerInfo.java: Loading...
  7. src/main/java/org/apache/hadoop/hbase/HServerLoad.java: Loading...
  8. src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java: Loading...
  9. src/main/java/org/apache/hadoop/hbase/MasterAddressTracker.java: Loading...
  10. src/main/java/org/apache/hadoop/hbase/Server.java: Loading...
  11. src/main/java/org/apache/hadoop/hbase/ServerName.java: Loading...
  12. src/main/java/org/apache/hadoop/hbase/avro/AvroUtil.java: Loading...
  13. src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java: Loading...
  14. src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java: Loading...
  15. src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java: Loading...
  16. src/main/java/org/apache/hadoop/hbase/catalog/RootLocationEditor.java: Loading...
  17. src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java: Loading...
  18. src/main/java/org/apache/hadoop/hbase/client/HConnection.java: Loading...
  19. src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java: Loading...
  20. src/main/java/org/apache/hadoop/hbase/client/HTable.java: Loading...
This diff has been split across 5 pages: 1 2 3 4 5 >