Review Board 1.7.22


Initial patch for RevisionManager endpoint.

Review Request #4313 - Created March 13, 2012 and updated

Thomas
HCATALOG-310
Reviewers
avandana, dcapwell, francisliu
hcatalog
Initial patch for RevisionManager endpoint.
unit tests pass
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
Revision 1300255 New Change
[20] 27 lines
[+20]
28
import java.util.Set;
28
import java.util.Set;
29

    
   
29

   
30
import org.apache.commons.logging.Log;
30
import org.apache.commons.logging.Log;
31
import org.apache.commons.logging.LogFactory;
31
import org.apache.commons.logging.LogFactory;
32
import org.apache.hadoop.conf.Configuration;
32
import org.apache.hadoop.conf.Configuration;
33
import org.apache.hadoop.hbase.HConstants;

   
34
import org.apache.hadoop.hive.hbase.HBaseSerDe;
33
import org.apache.hadoop.hive.hbase.HBaseSerDe;
35
import org.apache.hcatalog.common.HCatConstants;
34
import org.apache.hcatalog.common.HCatConstants;
36
import org.apache.hcatalog.common.HCatUtil;
35
import org.apache.hcatalog.common.HCatUtil;
37
import org.apache.hcatalog.data.schema.HCatFieldSchema;
36
import org.apache.hcatalog.data.schema.HCatFieldSchema;
38
import org.apache.hcatalog.data.schema.HCatSchema;
37
import org.apache.hcatalog.data.schema.HCatSchema;
39
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
38
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
40
import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory;
39
import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory;

    
   
40
import org.apache.hcatalog.hbase.snapshot.RevisionManagerProtocol;
41
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
41
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
42
import org.apache.hcatalog.hbase.snapshot.Transaction;
42
import org.apache.hcatalog.hbase.snapshot.Transaction;
43
import org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager;

   
44
import org.apache.hcatalog.mapreduce.HCatTableInfo;
43
import org.apache.hcatalog.mapreduce.HCatTableInfo;
45
import org.apache.hcatalog.mapreduce.InputJobInfo;
44
import org.apache.hcatalog.mapreduce.InputJobInfo;
46
import org.apache.hcatalog.mapreduce.OutputJobInfo;
45
import org.apache.hcatalog.mapreduce.OutputJobInfo;
47
import org.apache.hcatalog.mapreduce.StorerInfo;
46
import org.apache.hcatalog.mapreduce.StorerInfo;
48

    
   
47

   
[+20] [20] 75 lines
[+20] [+] static HCatTableSnapshot createSnapshot(Configuration jobConf,
124
     * @param jobConf The job configuration.
123
     * @param jobConf The job configuration.
125
     * @return RevisionManager An instance of revision manager.
124
     * @return RevisionManager An instance of revision manager.
126
     * @throws IOException
125
     * @throws IOException
127
     */
126
     */
128
    static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException {
127
    static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException {
129

    
   
128
      String factoryClassName = jobConf.get(RevisionManager.REVISION_MGR_CLIENT_FACTORY_CLASS);
130
        Properties properties = new Properties();
129
      // if a factory is set, we use it to create the RM client/proxy
131
        String zkHostList = jobConf.get(HConstants.ZOOKEEPER_QUORUM);
130
      if (factoryClassName != null) {
132
        int port = jobConf.getInt("hbase.zookeeper.property.clientPort",
131
        // at this time, this is the only factory we support, switch to dynamic instantiation later
133
                HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
132
        if (RevisionManagerProtocol.EndpointClientFactory.class.getName().equals(factoryClassName)) {
134

    
   
133
          return new RevisionManagerProtocol.EndpointClientFactory(jobConf).create(new Properties());
135
        if (zkHostList != null) {
134
        } else {
136
            String[] splits = zkHostList.split(",");
135
          throw new IllegalArgumentException("Unsupported "
137
            StringBuffer sb = new StringBuffer();
136
              + RevisionManager.REVISION_MGR_CLIENT_FACTORY_CLASS + ": " + factoryClassName);
138
            for (String split : splits) {
137
        }
139
                sb.append(split);
138
      }
140
                sb.append(':');
139
      // default behavior is to instantiate the RM implementation
141
                sb.append(port);
140
      return RevisionManagerFactory.getOpenedRevisionManager(jobConf);
142
                sb.append(',');

   
143
            }

   
144

    
   

   
145
            sb.deleteCharAt(sb.length() - 1);

   
146
            properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());

   
147
        }

   
148
        String dataDir = jobConf.get(ZKBasedRevisionManager.DATADIR);

   
149
        if (dataDir != null) {

   
150
            properties.put(ZKBasedRevisionManager.DATADIR, dataDir);

   
151
        }

   
152
        String rmClassName = jobConf.get(

   
153
                RevisionManager.REVISION_MGR_IMPL_CLASS,

   
154
                ZKBasedRevisionManager.class.getName());

   
155
        properties.put(RevisionManager.REVISION_MGR_IMPL_CLASS, rmClassName);

   
156
        RevisionManager revisionManger = RevisionManagerFactory

   
157
                .getRevisionManager(properties);

   
158
        revisionManger.open();

   
159
        return revisionManger;

   
160
    }
141
    }
161

    
   
142

   
162
    static void closeRevisionManagerQuietly(RevisionManager rm) {
143
    static void closeRevisionManagerQuietly(RevisionManager rm) {
163
        if (rm != null) {
144
        if (rm != null) {
164
            try {
145
            try {
[+20] [20] 125 lines
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
Revision 1300255 New Change
 
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
New File
 
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
Revision 1300255 New Change
 
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java
New File
 
trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
Revision 1300255 New Change
 
trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
Revision 1300255 New Change
 
trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
Revision 1300255 New Change
 
trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
New File
 
  1. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java: Loading...
  2. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java: Loading...
  3. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java: Loading...
  4. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java: Loading...
  5. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java: Loading...
  6. trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java: Loading...
  7. trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java: Loading...
  8. trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java: Loading...
  9. trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java: Loading...