Review Board 1.7.22


FLUME-1844: HDFSEventSink should have option to use RawLocalFileSystem

Review Request #8964 - Created Jan. 15, 2013 and submitted

Brock Noland
trunk
FLUME-1844
Reviewers
Flume
hshreedharan, juhanic
flume-git
Add's flag to HDFSEventSink Serializer's to use raw local file system.
Unit tests pass.
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java
Revision f14f7cb New Change
[20] 31 lines
[+20]
32
import org.apache.flume.SystemClock;
32
import org.apache.flume.SystemClock;
33
import org.apache.flume.instrumentation.SinkCounter;
33
import org.apache.flume.instrumentation.SinkCounter;
34
import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
34
import org.apache.flume.sink.hdfs.HDFSEventSink.WriterCallback;
35
import org.apache.hadoop.conf.Configuration;
35
import org.apache.hadoop.conf.Configuration;
36
import org.apache.hadoop.fs.FileSystem;
36
import org.apache.hadoop.fs.FileSystem;

    
   
37
import org.apache.hadoop.fs.LocalFileSystem;
37
import org.apache.hadoop.fs.Path;
38
import org.apache.hadoop.fs.Path;
38
import org.apache.hadoop.io.SequenceFile.CompressionType;
39
import org.apache.hadoop.io.SequenceFile.CompressionType;
39
import org.apache.hadoop.io.compress.CompressionCodec;
40
import org.apache.hadoop.io.compress.CompressionCodec;
40
import org.apache.hadoop.security.UserGroupInformation;
41
import org.apache.hadoop.security.UserGroupInformation;
41
import org.slf4j.Logger;
42
import org.slf4j.Logger;
[+20] [20] 84 lines
[+20] [+] class BucketWriter {
126
    this.onIdleCallbackPath = onIdleCallbackPath;
127
    this.onIdleCallbackPath = onIdleCallbackPath;
127

    
   
128

   
128
    fileExtensionCounter = new AtomicLong(clock.currentTimeMillis());
129
    fileExtensionCounter = new AtomicLong(clock.currentTimeMillis());
129

    
   
130

   
130
    isOpen = false;
131
    isOpen = false;
131
    writer.configure(context);
132
    this.writer.configure(context);
132
  }
133
  }
133

    
   
134

   
134
  /**
135
  /**
135
   * Allow methods to act as another user (typically used for HDFS Kerberos)
136
   * Allow methods to act as another user (typically used for HDFS Kerberos)
136
   * @param <T>
137
   * @param <T>
[+20] [20] 356 lines
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java
Revision 18fe6d4 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java
Revision bd40a88 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
Revision 1e6d68f New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java
Revision 1035ac3 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java
New File
 
  1. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/BucketWriter.java: Loading...
  2. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSCompressedDataStream.java: Loading...
  3. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSDataStream.java: Loading...
  4. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java: Loading...
  5. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSink.java: Loading...
  6. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestUseRawLocalFileSystem.java: Loading...