Review Board 1.7.22


[HBase-4120] isolation and allocation,patch for table priority, test cases are added too.

Review Request #1421 - Created Aug. 9, 2011 and updated

Jia Liu
trunk
HBase-4120
Reviewers
hbase
hbase
Patch used for table priority alone,In this patch, not only tables can have different priorities but also the different actions like "get","scan","put" and "delete" can have priorities.
Tested with test cases in  TestCase_For_TablePriority_trunk_v1.patch 
please apply the patch of HBASE-4181 first,in some circumstances this bug will affect the performance of client.
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
Diff Revision 17 Diff Revision 20
1
/*
1
/*
2
 * Copyright 2011 The Apache Software Foundation
2
 * Copyright 2011 The Apache Software Foundation
3
 *
3
 *
4
 * Licensed to the Apache Software Foundation (ASF) under one
4
 * Licensed to the Apache Software Foundation (ASF) under one
5
 * or more contributor license agreements.  See the NOTICE file
5
 * or more contributor license agreements.  See the NOTICE file
6
 * distributed with this work for additional information
6
 * distributed with this work for additional information
7
 * regarding copyright ownership.  The ASF licenses this file
7
 * regarding copyright ownership.  The ASF licenses this file
8
 * to you under the Apache License, Version 2.0 (the
8
 * to you under the Apache License, Version 2.0 (the
9
 * "License"); you may not use this file except in compliance
9
 * "License"); you may not use this file except in compliance
10
 * with the License.  You may obtain a copy of the License at
10
 * with the License.  You may obtain a copy of the License at
11
 *
11
 *
12
 *     http://www.apache.org/licenses/LICENSE-2.0
12
 *     http://www.apache.org/licenses/LICENSE-2.0
13
 *
13
 *
14
 * Unless required by applicable law or agreed to in writing, software
14
 * Unless required by applicable law or agreed to in writing, software
15
 * distributed under the License is distributed on an "AS IS" BASIS,
15
 * distributed under the License is distributed on an "AS IS" BASIS,
16
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
 * See the License for the specific language governing permissions and
17
 * See the License for the specific language governing permissions and
18
 * limitations under the License.
18
 * limitations under the License.
19
 */
19
 */
20

    
   
20

   
21
package org.apache.hadoop.hbase.client;
21
package org.apache.hadoop.hbase.client;
22

    
   
22

   
23
import java.util.ArrayList;
23
import java.util.ArrayList;
24
import java.util.HashMap;
24
import java.util.HashMap;
25
import java.util.List;
25
import java.util.List;
26
import java.util.Map;
26
import java.util.Map;
27
import java.util.TreeMap;
27
import java.util.TreeMap;
28
import java.util.UUID;
28
import java.util.UUID;
29

    
   
29

   
30
import org.apache.hadoop.hbase.HConstants;
30
import org.apache.hadoop.hbase.HConstants;
31
import org.apache.hadoop.hbase.KeyValue;
31
import org.apache.hadoop.hbase.KeyValue;
32
import org.apache.hadoop.hbase.ipc.PriorityJobQueue.ActionPriorities;
32
import org.apache.hadoop.hbase.ipc.PriorityJobQueue.ActionPriorities;
33
import org.apache.hadoop.hbase.util.Bytes;
33
import org.apache.hadoop.hbase.util.Bytes;
34

    
   
34

   
35
public abstract class Mutation extends OperationWithAttributes {
35
public abstract class Mutation extends OperationWithAttributes {
36
  // Attribute used in Mutations to indicate the originating cluster.
36
  // Attribute used in Mutations to indicate the originating cluster.
37
  private static final String CLUSTER_ID_ATTR = "_c.id_";
37
  private static final String CLUSTER_ID_ATTR = "_c.id_";
38

    
   
38

   
39
  protected byte [] row = null;
39
  protected byte [] row = null;
40
  protected long ts = HConstants.LATEST_TIMESTAMP;
40
  protected long ts = HConstants.LATEST_TIMESTAMP;
41
  protected long lockId = -1L;
41
  protected long lockId = -1L;
42
  protected boolean writeToWAL = true;
42
  protected boolean writeToWAL = true;
43
  protected Map<byte [], List<KeyValue>> familyMap =
43
  protected Map<byte [], List<KeyValue>> familyMap =
44
      new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
44
      new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
45

    
   
45

   
46
  /**
46
  /**
47
   * Compile the column family (i.e. schema) information
47
   * Compile the column family (i.e. schema) information
48
   * into a Map. Useful for parsing and aggregation by debugging,
48
   * into a Map. Useful for parsing and aggregation by debugging,
49
   * logging, and administration tools.
49
   * logging, and administration tools.
50
   * @return Map
50
   * @return Map
51
   */
51
   */
52
  @Override
52
  @Override
53
  public Map<String, Object> getFingerprint() {
53
  public Map<String, Object> getFingerprint() {
54
    Map<String, Object> map = new HashMap<String, Object>();
54
    Map<String, Object> map = new HashMap<String, Object>();
55
    List<String> families = new ArrayList<String>();
55
    List<String> families = new ArrayList<String>();
56
    // ideally, we would also include table information, but that information
56
    // ideally, we would also include table information, but that information
57
    // is not stored in each Operation instance.
57
    // is not stored in each Operation instance.
58
    map.put("families", families);
58
    map.put("families", families);
59
    for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
59
    for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
60
      families.add(Bytes.toStringBinary(entry.getKey()));
60
      families.add(Bytes.toStringBinary(entry.getKey()));
61
    } 
61
    } 
62
    return map;
62
    return map;
63
  }
63
  }
64

    
   
64

   
65
  /**
65
  /**
66
   * Compile the details beyond the scope of getFingerprint (row, columns,
66
   * Compile the details beyond the scope of getFingerprint (row, columns,
67
   * timestamps, etc.) into a Map along with the fingerprinted information.
67
   * timestamps, etc.) into a Map along with the fingerprinted information.
68
   * Useful for debugging, logging, and administration tools.
68
   * Useful for debugging, logging, and administration tools.
69
   * @param maxCols a limit on the number of columns output prior to truncation
69
   * @param maxCols a limit on the number of columns output prior to truncation
70
   * @return Map
70
   * @return Map
71
   */
71
   */
72
  @Override
72
  @Override
73
  public Map<String, Object> toMap(int maxCols) {
73
  public Map<String, Object> toMap(int maxCols) {
74
    // we start with the fingerprint map and build on top of it.
74
    // we start with the fingerprint map and build on top of it.
75
    Map<String, Object> map = getFingerprint();
75
    Map<String, Object> map = getFingerprint();
76
    // replace the fingerprint's simple list of families with a 
76
    // replace the fingerprint's simple list of families with a 
77
    // map from column families to lists of qualifiers and kv details
77
    // map from column families to lists of qualifiers and kv details
78
    Map<String, List<Map<String, Object>>> columns =
78
    Map<String, List<Map<String, Object>>> columns =
79
      new HashMap<String, List<Map<String, Object>>>();
79
      new HashMap<String, List<Map<String, Object>>>();
80
    map.put("families", columns);
80
    map.put("families", columns);
81
    map.put("row", Bytes.toStringBinary(this.row));
81
    map.put("row", Bytes.toStringBinary(this.row));
82
    int colCount = 0;
82
    int colCount = 0;
83
    // iterate through all column families affected
83
    // iterate through all column families affected
84
    for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
84
    for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
85
      // map from this family to details for each kv affected within the family
85
      // map from this family to details for each kv affected within the family
86
      List<Map<String, Object>> qualifierDetails =
86
      List<Map<String, Object>> qualifierDetails =
87
        new ArrayList<Map<String, Object>>();
87
        new ArrayList<Map<String, Object>>();
88
      columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
88
      columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
89
      colCount += entry.getValue().size();
89
      colCount += entry.getValue().size();
90
      if (maxCols <= 0) {
90
      if (maxCols <= 0) {
91
        continue;
91
        continue;
92
      }
92
      }
93
      // add details for each kv
93
      // add details for each kv
94
      for (KeyValue kv : entry.getValue()) {
94
      for (KeyValue kv : entry.getValue()) {
95
        if (--maxCols <= 0 ) {
95
        if (--maxCols <= 0 ) {
96
          continue;
96
          continue;
97
        }
97
        }
98
        Map<String, Object> kvMap = kv.toStringMap();
98
        Map<String, Object> kvMap = kv.toStringMap();
99
        // row and family information are already available in the bigger map
99
        // row and family information are already available in the bigger map
100
        kvMap.remove("row");
100
        kvMap.remove("row");
101
        kvMap.remove("family");
101
        kvMap.remove("family");
102
        qualifierDetails.add(kvMap);
102
        qualifierDetails.add(kvMap);
103
      }
103
      }
104
    }
104
    }
105
    map.put("totalColumns", colCount);
105
    map.put("totalColumns", colCount);
106
    return map;
106
    return map;
107
  }
107
  }
108

    
   
108

   
109
  /**
109
  /**
110
   * @return true if edits should be applied to WAL, false if not
110
   * @return true if edits should be applied to WAL, false if not
111
   */
111
   */
112
  public boolean getWriteToWAL() {
112
  public boolean getWriteToWAL() {
113
    return this.writeToWAL;
113
    return this.writeToWAL;
114
  }
114
  }
115

    
   
115

   
116
  /**
116
  /**
117
   * Set whether this Delete should be written to the WAL or not.
117
   * Set whether this Delete should be written to the WAL or not.
118
   * Not writing the WAL means you may lose edits on server crash.
118
   * Not writing the WAL means you may lose edits on server crash.
119
   * @param write true if edits should be written to WAL, false if not
119
   * @param write true if edits should be written to WAL, false if not
120
   */
120
   */
121
  public void setWriteToWAL(boolean write) {
121
  public void setWriteToWAL(boolean write) {
122
    this.writeToWAL = write;
122
    this.writeToWAL = write;
123
  }
123
  }
124

    
   
124

   
125
  /**
125
  /**
126
   * Method for retrieving the put's familyMap
126
   * Method for retrieving the put's familyMap
127
   * @return familyMap
127
   * @return familyMap
128
   */
128
   */
129
  public Map<byte [], List<KeyValue>> getFamilyMap() {
129
  public Map<byte [], List<KeyValue>> getFamilyMap() {
130
    return this.familyMap;
130
    return this.familyMap;
131
  }
131
  }
132

    
   
132

   
133
  /**
133
  /**
134
   * Method for setting the put's familyMap
134
   * Method for setting the put's familyMap
135
   */
135
   */
136
  public void setFamilyMap(Map<byte [], List<KeyValue>> map) {
136
  public void setFamilyMap(Map<byte [], List<KeyValue>> map) {
137
    this.familyMap = map;
137
    this.familyMap = map;
138
  }
138
  }
139

    
   
139

   
140
  /**
140
  /**
141
   * Method to check if the familyMap is empty
141
   * Method to check if the familyMap is empty
142
   * @return true if empty, false otherwise
142
   * @return true if empty, false otherwise
143
   */
143
   */
144
  public boolean isEmpty() {
144
  public boolean isEmpty() {
145
    return familyMap.isEmpty();
145
    return familyMap.isEmpty();
146
  }
146
  }
147

    
   
147

   
148
  /**
148
  /**
149
   * Method for retrieving the delete's row
149
   * Method for retrieving the delete's row
150
   * @return row
150
   * @return row
151
   */
151
   */
152
  public byte [] getRow() {
152
  public byte [] getRow() {
153
    return this.row;
153
    return this.row;
154
  }
154
  }
155

    
   
155

   
156
  public int compareTo(final Row d) {
156
  public int compareTo(final Row d) {
157
    return Bytes.compareTo(this.getRow(), d.getRow());
157
    return Bytes.compareTo(this.getRow(), d.getRow());
158
  }
158
  }
159

    
   
159

   
160
  /**
160
  /**
161
   * Method for retrieving the delete's RowLock
161
   * Method for retrieving the delete's RowLock
162
   * @return RowLock
162
   * @return RowLock
163
   */
163
   */
164
  public RowLock getRowLock() {
164
  public RowLock getRowLock() {
165
    return new RowLock(this.row, this.lockId);
165
    return new RowLock(this.row, this.lockId);
166
  }
166
  }
167

    
   
167

   
168
  /**
168
  /**
169
   * Method for retrieving the delete's lock ID.
169
   * Method for retrieving the delete's lock ID.
170
   *
170
   *
171
   * @return The lock ID.
171
   * @return The lock ID.
172
   */
172
   */
173
  public long getLockId() {
173
  public long getLockId() {
174
  return this.lockId;
174
  return this.lockId;
175
  }
175
  }
176

    
   
176

   
177
  /**
177
  /**
178
   * Method for retrieving the timestamp
178
   * Method for retrieving the timestamp
179
   * @return timestamp
179
   * @return timestamp
180
   */
180
   */
181
  public long getTimeStamp() {
181
  public long getTimeStamp() {
182
    return this.ts;
182
    return this.ts;
183
  }
183
  }
184

    
   
184

   
185
  /**
185
  /**
186
   * Set the replication custer id.
186
   * Set the replication custer id.
187
   * @param clusterId
187
   * @param clusterId
188
   */
188
   */
189
  public void setClusterId(UUID clusterId) {
189
  public void setClusterId(UUID clusterId) {
190
    byte[] val = new byte[2*Bytes.SIZEOF_LONG];
190
    byte[] val = new byte[2*Bytes.SIZEOF_LONG];
191
    Bytes.putLong(val, 0, clusterId.getMostSignificantBits());
191
    Bytes.putLong(val, 0, clusterId.getMostSignificantBits());
192
    Bytes.putLong(val, Bytes.SIZEOF_LONG, clusterId.getLeastSignificantBits());
192
    Bytes.putLong(val, Bytes.SIZEOF_LONG, clusterId.getLeastSignificantBits());
193
    setAttribute(CLUSTER_ID_ATTR, val);
193
    setAttribute(CLUSTER_ID_ATTR, val);
194
  }
194
  }
195

    
   
195

   
196
  /**
196
  /**
197
   * @return The replication cluster id.
197
   * @return The replication cluster id.
198
   */
198
   */
199
  public UUID getClusterId() {
199
  public UUID getClusterId() {
200
    byte[] attr = getAttribute(CLUSTER_ID_ATTR);
200
    byte[] attr = getAttribute(CLUSTER_ID_ATTR);
201
    if (attr == null) {
201
    if (attr == null) {
202
      return HConstants.DEFAULT_CLUSTER_ID;
202
      return HConstants.DEFAULT_CLUSTER_ID;
203
    }
203
    }
204
    return new UUID(Bytes.toLong(attr,0), Bytes.toLong(attr, Bytes.SIZEOF_LONG));
204
    return new UUID(Bytes.toLong(attr,0), Bytes.toLong(attr, Bytes.SIZEOF_LONG));
205
  }
205
  }
206

    
   
206

   
207
  /**
207
  /**
208
   * @return the total number of KeyValues
208
   * @return the total number of KeyValues
209
   */
209
   */
210
  public int size() {
210
  public int size() {
211
    int size = 0;
211
    int size = 0;
212
    for(List<KeyValue> kvList : this.familyMap.values()) {
212
    for(List<KeyValue> kvList : this.familyMap.values()) {
213
      size += kvList.size();
213
      size += kvList.size();
214
    }
214
    }
215
    return size;
215
    return size;
216
  }
216
  }
217

    
   
217

   
218
  /**
218
  /**
219
   * @return the number of different families
219
   * @return the number of different families
220
   */
220
   */
221
  public int numFamilies() {
221
  public int numFamilies() {
222
    return familyMap.size();
222
    return familyMap.size();
223
  }
223
  }
224
  
224
  
225

    
   
225

   
226
  @Override
226
  @Override
227
  public int getPriority(ActionPriorities actionPriority) {
227
  public int getPriority(ActionPriorities actionPriority) {
228
	  return actionPriority.getGetPriority();
228
	  return actionPriority.getPutPriority();
229
  }
229
  }
230
}
230
}
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityFunction.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityHBaseServer.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityJobQueue.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/QosRegionObserver.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestActionPriority.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestPriorityJobQueue.java
Diff Revision 17 Diff Revision 20
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestTablePriority.java
Diff Revision 17 Diff Revision 20
 
  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Mutation.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityFunction.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityHBaseServer.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/PriorityJobQueue.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/QosRegionObserver.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestActionPriority.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestPriorityJobQueue.java: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ipc/TestTablePriority.java: Loading...