Review Board 1.7.22


FLUME-1865. Rename the Sequence File formatters to Serializer to be consistent with the rest of Flume

Review Request #9074 - Created Jan. 23, 2013 and submitted

Hari Shreedharan
FLUME-1865
Reviewers
Flume
flume-git
Mostly an IDE based refactoring, to change formatters to serializers and format method to serialize method. I was confused whether we should move them into a different package (if yes, what it should be called!).
Ran full tests.

Diff revision 1 (Latest)

  1. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java: Loading...
  2. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextFormatter.java: Loading...
  3. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java: Loading...
  4. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableFormatter.java: Loading...
  5. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java: Loading...
  6. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatter.java: Loading...
  7. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterFactory.java: Loading...
  8. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterType.java: Loading...
  9. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java: Loading...
  10. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java: Loading...
  11. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java: Loading...
  12. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomFormatter.java: Loading...
  13. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java: Loading...
  14. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java: Loading...
  15. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java: Loading...
  16. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSeqFileFormatterFactory.java: Loading...
  17. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java: Loading...
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java
Revision e127f6a New Change
[20] 36 lines
[+20] [+] public class HDFSSequenceFile implements HDFSWriter {
37
  private static final Logger logger =
37
  private static final Logger logger =
38
      LoggerFactory.getLogger(HDFSSequenceFile.class);
38
      LoggerFactory.getLogger(HDFSSequenceFile.class);
39
  private SequenceFile.Writer writer;
39
  private SequenceFile.Writer writer;
40
  private String writeFormat;
40
  private String writeFormat;
41
  private Context serializerContext;
41
  private Context serializerContext;
42
  private SeqFileFormatter formatter;
42
  private SequenceFileSerializer serializer;
43
  private boolean useRawLocalFileSystem;
43
  private boolean useRawLocalFileSystem;
44

    
   
44

   
45
  public HDFSSequenceFile() {
45
  public HDFSSequenceFile() {
46
    writer = null;
46
    writer = null;
47
  }
47
  }
48

    
   
48

   
49
  @Override
49
  @Override
50
  public void configure(Context context) {
50
  public void configure(Context context) {
51
    // use binary writable format by default
51
    // use binary writable serialize by default
52
    writeFormat = context.getString("hdfs.writeFormat", SeqFileFormatterType.Writable.name());
52
    writeFormat = context.getString("hdfs.writeFormat",

    
   
53
      SequenceFileSerializerType.Writable.name());
53
    useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
54
    useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
54
        false);
55
        false);
55
    serializerContext = new Context(
56
    serializerContext = new Context(
56
            context.getSubProperties(SeqFileFormatterFactory.CTX_PREFIX));
57
            context.getSubProperties(SequenceFileSerializerFactory.CTX_PREFIX));
57
    formatter = SeqFileFormatterFactory
58
    serializer = SequenceFileSerializerFactory
58
            .getFormatter(writeFormat, serializerContext);
59
            .getSerializer(writeFormat, serializerContext);
59
    logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
60
    logger.info("writeFormat = " + writeFormat + ", UseRawLocalFileSystem = "
60
        + useRawLocalFileSystem);
61
        + useRawLocalFileSystem);
61
  }
62
  }
62

    
   
63

   
63
  @Override
64
  @Override
[+20] [20] 16 lines
[+20] [+] public void open(String filePath, CompressionCodec codeC,
80
      }
81
      }
81
    }
82
    }
82
    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
83
    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
83
            (dstPath)) {
84
            (dstPath)) {
84
      FSDataOutputStream outStream = hdfs.append(dstPath);
85
      FSDataOutputStream outStream = hdfs.append(dstPath);
85
      writer = SequenceFile.createWriter(conf, outStream, formatter.getKeyClass(),
86
      writer = SequenceFile.createWriter(conf, outStream, serializer
86
          formatter.getValueClass(), compType, codeC);
87
        .getKeyClass(),

    
   
88
        serializer.getValueClass(), compType, codeC);
87
    } else {
89
    } else {
88
      writer = SequenceFile.createWriter(hdfs, conf, dstPath,
90
      writer = SequenceFile.createWriter(hdfs, conf, dstPath,
89
          formatter.getKeyClass(), formatter.getValueClass(), compType, codeC);
91
        serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);
90
    }
92
    }
91
  }
93
  }
92

    
   
94

   
93
  @Override
95
  @Override
94
  public void append(Event e) throws IOException {
96
  public void append(Event e) throws IOException {
95
    for (SeqFileFormatter.Record record : formatter.format(e)) {
97
    for (SequenceFileSerializer.Record record : serializer.serialize(e)) {
96
      writer.append(record.getKey(), record.getValue());
98
      writer.append(record.getKey(), record.getValue());
97
    }
99
    }
98
  }
100
  }
99

    
   
101

   
100
  @Override
102
  @Override
[+20] [20] 9 lines
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextFormatter.java
Revision 4b39f5d New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableFormatter.java
Revision cece506 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatter.java
Revision c25931c New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterFactory.java
Revision 20409ba New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterType.java
Revision ff3eb84 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomFormatter.java
Revision ab1e463 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java
New File
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java
Revision 829d7e8 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java
Revision bcd19e9 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSeqFileFormatterFactory.java
Revision 9d17785 New Change
 
flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java
New File
 
  1. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSSequenceFile.java: Loading...
  2. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextFormatter.java: Loading...
  3. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSTextSerializer.java: Loading...
  4. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableFormatter.java: Loading...
  5. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/HDFSWritableSerializer.java: Loading...
  6. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatter.java: Loading...
  7. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterFactory.java: Loading...
  8. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SeqFileFormatterType.java: Loading...
  9. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializer.java: Loading...
  10. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerFactory.java: Loading...
  11. flume-ng-sinks/flume-hdfs-sink/src/main/java/org/apache/flume/sink/hdfs/SequenceFileSerializerType.java: Loading...
  12. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomFormatter.java: Loading...
  13. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/MyCustomSerializer.java: Loading...
  14. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestBucketWriter.java: Loading...
  15. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestHDFSEventSinkOnMiniCluster.java: Loading...
  16. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSeqFileFormatterFactory.java: Loading...
  17. flume-ng-sinks/flume-hdfs-sink/src/test/java/org/apache/flume/sink/hdfs/TestSequenceFileSerializerFactory.java: Loading...