Review Board 1.7.22


HBASE-2195 - Support cyclic replication

Review Request #1730 - Created Sept. 6, 2011 and submitted

Lars Hofhansl
trunk
HBASE-2195
Reviewers
hbase
jdcryans, stack, tedyu
hbase
For Master <-> Master replication (as well as cycles > 2) a replication sink needs to keep track who is was the originator of a change and
(1) do not replicate this data back to the originator and
(2) pass that information on to the next link in a cycle > 2

In order to do that, HlogKeys read from the WAL are tagged with the Cluster UUID at the ReplicationSource before the logs are shipped to the Sink. The sink writes the Cluster UUID into the WAL log (in HlogKey, so only once per edit). If the sink is also source for yet another sink the ClusterUUID is retained in the HLogKeys read from the local WAL and passed on to the sink.
All tests pass.
New test: TestMasterReplication (which tests Master <-> Master, but no cycles > 2, yet), also passes.
Manually tested a cycle between 3 clusters.

Diff revision 2

This is not the most recent revision of the diff. The latest diff is revision 5. See what's changed.

1 2 3 4 5
1 2 3 4 5

  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Delete.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Put.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java: Loading...
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
Revision 1165864 New Change
[20] 22 lines
[+20]
23
import org.apache.hadoop.hbase.util.Bytes;
23
import org.apache.hadoop.hbase.util.Bytes;
24

    
   
24

   
25
import java.util.ArrayList;
25
import java.util.ArrayList;
26
import java.util.Arrays;
26
import java.util.Arrays;
27
import java.util.List;
27
import java.util.List;

    
   
28
import java.util.UUID;
28
import java.util.regex.Pattern;
29
import java.util.regex.Pattern;
29

    
   
30

   
30
/**
31
/**
31
 * HConstants holds a bunch of HBase-related constants
32
 * HConstants holds a bunch of HBase-related constants
32
 */
33
 */
[+20] [20] 168 lines
[+20]
201
  public static final String CLUSTER_ID_FILE_NAME = "hbase.id";
202
  public static final String CLUSTER_ID_FILE_NAME = "hbase.id";
202

    
   
203

   
203
  /** Configuration key storing the cluster ID */
204
  /** Configuration key storing the cluster ID */
204
  public static final String CLUSTER_ID = "hbase.cluster.id";
205
  public static final String CLUSTER_ID = "hbase.cluster.id";
205

    
   
206

   

    
   
207
  /**

    
   
208
   * Attribute used in Puts and Gets to indicate the originating

    
   
209
   * cluster.

    
   
210
   */

    
   
211
  public static final String CLUSTER_ID_ATTR = "_c.id_";

    
   
212

   
206
  // Always store the location of the root table's HRegion.
213
  // Always store the location of the root table's HRegion.
207
  // This HRegion is never split.
214
  // This HRegion is never split.
208

    
   
215

   
209
  // region name = table + startkey + regionid. This is the row key.
216
  // region name = table + startkey + regionid. This is the row key.
210
  // each row in the root and meta tables describes exactly 1 region
217
  // each row in the root and meta tables describes exactly 1 region
[+20] [20] 151 lines
[+20]
362

    
   
369

   
363
  /**
370
  /**
364
   * Default cluster ID, cannot be used to identify a cluster so a key with
371
   * Default cluster ID, cannot be used to identify a cluster so a key with
365
   * this value means it wasn't meant for replication.
372
   * this value means it wasn't meant for replication.
366
   */
373
   */
367
  public static final byte DEFAULT_CLUSTER_ID = 0;
374
  public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L,0L);
368

    
   
375

   
369
    /**
376
    /**
370
     * Parameter name for maximum number of bytes returned when calling a
377
     * Parameter name for maximum number of bytes returned when calling a
371
     * scanner's next method.
378
     * scanner's next method.
372
     */
379
     */
[+20] [20] 142 lines
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Delete.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Put.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Revision 1165864 New Change
 
http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java
New File
 
  1. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java: Loading...
  2. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Delete.java: Loading...
  3. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Put.java: Loading...
  4. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java: Loading...
  5. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  6. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java: Loading...
  7. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java: Loading...
  8. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java: Loading...
  9. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java: Loading...
  10. http://svn.apache.org/repos/asf/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java: Loading...
  11. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java: Loading...
  12. http://svn.apache.org/repos/asf/hbase/trunk/src/test/java/org/apache/hadoop/hbase/replication/TestMasterReplication.java: Loading...