Review Board 1.7.22


hbase-5453 Switch on-disk formats (reference files, HFile meta fields, etc) to PB

Review Request #5130 - Created May 15, 2012 and updated

Michael Stack
hbase-5453
Reviewers
hbase
hbase-git
A b/src/main/java/org/apache/hadoop/hbase/ClusterId.java
  New  class to hold clusterid in.
M b/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
  Make it so can do pb serialization.  Deprecated Writable serialization.
M b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
  Make it so methods in here follow the pattern in HCD an HTD pb 'ing.
  Deprecated Writable serialization.
M b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
  Make it so can do pb serialization.  Deprecated Writable serialization.
M b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
  ClusterId under ZK got renamed as ZKClusterId
M b/src/main/java/org/apache/hadoop/hbase/io/Reference.java
  Hide the Reference#Range enums.  Don't let them out of this class.
  Make it so can do pb serialization.
M b/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
  Use new methods on Reference for getting top and bottom.
M b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
  ClusterId under zk has been renamed ZKClusterId.
  Use new ClusterId class too.
M b/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
  Use new clusterid class.
M b/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
  Move the RegionInfo convertion up into HRegionInfo instead of here.
  Added generic toDelimitedByteArray helper.
M b/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
  Use HRegionInfo convertions instead.
M b/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java
  Use HRegionInfo convertions instead.
M b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
  Use new utility writing out .regioninfo files.
M b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
  Formatting.
M b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
M b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
  Range in Reference is no longer public.
  Range in Reference is no longer public.
M b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
M b/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
  ClusterId got renamed ZKClusterId
M b/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
  Use new serialization utlity in HTD.
M  b/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
  Generic method for writing dot file content.
M b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
  Reference#Range is not public any more
M b/src/main/java/org/apache/hadoop/hbase/util/Writables.java
  Deprecated getHRegionInfo, etc.
D b/src/main/java/org/apache/hadoop/hbase/zookeeper/ClusterId.java
A b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKClusterId.java
  Rename
A b/src/main/protobuf/ClusterId.proto
  Added file for ClusterId only since its written to fs and to zk.
A b/src/main/protobuf/FS.proto
  Protos for fs files.
M b/src/main/protobuf/ZooKeeper.proto
  Moved ClusterId out to own proto file
M b/src/main/protobuf/hbase.proto
  Added TableSchema and ColumnFamilySchema

 
src/main/java/org/apache/hadoop/hbase/ClusterId.java
Diff Revision 1 Diff Revision 2
[20] 15 lines
[+20]
16
 * limitations under the License.
16
 * limitations under the License.
17
 */
17
 */
18

    
   
18

   
19
package org.apache.hadoop.hbase;
19
package org.apache.hadoop.hbase;
20

    
   
20

   
21
import java.io.BufferedInputStream;

   
22
import java.io.IOException;

   
23
import java.io.InputStream;

   
24
import java.util.UUID;
21
import java.util.UUID;
25

    
   
22

   
26
import org.apache.hadoop.classification.InterfaceAudience;
23
import org.apache.hadoop.classification.InterfaceAudience;
27
import org.apache.hadoop.fs.FSDataInputStream;

   
28
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
24
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
29
import org.apache.hadoop.hbase.protobuf.generated.ClusterIdProtos;
25
import org.apache.hadoop.hbase.protobuf.generated.ClusterIdProtos;
30
import org.apache.hadoop.hbase.util.Bytes;
26
import org.apache.hadoop.hbase.util.Bytes;
31

    
   
27

   
32
import com.google.protobuf.InvalidProtocolBufferException;
28
import com.google.protobuf.InvalidProtocolBufferException;
[+20] [20] 41 lines
[+20] [+] public static ClusterId parseFrom(final byte [] bytes) throws DeserializationException {
74
      } catch (InvalidProtocolBufferException e) {
70
      } catch (InvalidProtocolBufferException e) {
75
        throw new DeserializationException(e);
71
        throw new DeserializationException(e);
76
      }
72
      }
77
      return convert(cid);
73
      return convert(cid);
78
    } else {
74
    } else {

    
   
75
      // Presume it was written out this way, the old way.
79
      return new ClusterId(Bytes.toString(bytes));
76
      return new ClusterId(Bytes.toString(bytes));
80
    }
77
    }
81
  }
78
  }
82

    
   
79

   
83
  /**
80
  /**
84
   * Parses a {@link ClusterId} instance from the passed in stream.  Presumes the stream contains

   
85
   * nothing but a serialized clusterid.

   
86
   * @param in

   
87
   * @return An instance of ClusterId.

   
88
   * @throws IOException

   
89
   */

   
90
  public static ClusterId parseFrom(final FSDataInputStream in) throws IOException {

   
91
    // I need to be able to move back in the stream if this is not a pb serialization so I can

   
92
    // do the Writable decoding instead.

   
93
    InputStream is = in.markSupported()? in: new BufferedInputStream(in);

   
94
    int pblen = ProtobufUtil.lengthOfPBMagic();

   
95
    is.mark(pblen);

   
96
    byte [] pbuf = new byte[pblen];

   
97
    int read = is.read(pbuf);

   
98
    if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen);

   
99
    if (ProtobufUtil.isPBMagicPrefix(pbuf)) {

   
100
      return convert(ClusterIdProtos.ClusterId.parseFrom(is));

   
101
    } else {

   
102
      // Presume Writables.  Need to reset the stream since it didn't start w/ pb.

   
103
      in.reset();

   
104
      String cid = in.readUTF();

   
105
      return new ClusterId(cid);

   
106
    }

   
107
  }

   
108

    
   

   
109
  /**

   
110
   * @return A pb instance to represent this instance.
81
   * @return A pb instance to represent this instance.
111
   */
82
   */
112
  ClusterIdProtos.ClusterId convert() {
83
  ClusterIdProtos.ClusterId convert() {
113
    ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();
84
    ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();
114
    return builder.setClusterId(this.id).build();
85
    return builder.setClusterId(this.id).build();
[+20] [20] 10 lines
src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/io/Reference.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
Diff Revision 1 Diff Revision 2
 
src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
Diff Revision 1 Diff Revision 2
 
src/main/protobuf/FS.proto
Diff Revision 1 Diff Revision 2
 
src/main/protobuf/hbase.proto
Diff Revision 1 Diff Revision 2
 
src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
Diff Revision 1 Diff Revision 2
 
  1. src/main/java/org/apache/hadoop/hbase/ClusterId.java: Loading...
  2. src/main/java/org/apache/hadoop/hbase/HRegionInfo.java: Loading...
  3. src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java: Loading...
  4. src/main/java/org/apache/hadoop/hbase/io/Reference.java: Loading...
  5. src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java: Loading...
  6. src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java: Loading...
  7. src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java: Loading...
  8. src/main/java/org/apache/hadoop/hbase/util/FSUtils.java: Loading...
  9. src/main/protobuf/FS.proto: Loading...
  10. src/main/protobuf/hbase.proto: Loading...
  11. src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java: Loading...