Review Board 1.7.22


FLUME-1586. File Channel should support verifying integrity of individual events.

Review Request #10944 - Created May 6, 2013 and submitted

Hari Shreedharan
FLUME-1586
Reviewers
Flume
flume-git
Patch to add a checksum to events and replace them with a noop event using a tool, if corrupt.
Added unit tests when corrupt and noop events are encountered. I will add tests for the tool as well soon. I have not yet tested the tool completely. This patch aims at gathering feedback on the approach.

Diff revision 10 (Latest)

1 2 3 4 5 6 7 8 9 10
1 2 3 4 5 6 7 8 9 10

  1. pom.xml: Loading...
  2. bin/flume-ng: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java: Loading...
  7. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  8. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java: Loading...
  9. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java: Loading...
  10. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java: Loading...
  11. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java: Loading...
  12. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java: Loading...
  13. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java: Loading...
  14. flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto: Loading...
  15. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java: Loading...
  16. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java: Loading...
  17. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java: Loading...
  18. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java: Loading...
  19. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java: Loading...
  20. flume-ng-dist/pom.xml: Loading...
  21. flume-ng-dist/src/main/assembly/bin.xml: Loading...
  22. flume-ng-dist/src/main/assembly/src.xml: Loading...
  23. flume-tools/pom.xml: Loading...
  24. flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java: Loading...
  25. flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java: Loading...
  26. flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java: Loading...
  27. flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java: Loading...
  28. flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java: Loading...
pom.xml
Revision 9982f5d New Change
[20] 60 lines
[+20]
61
    <module>flume-ng-channels</module>
61
    <module>flume-ng-channels</module>
62
    <module>flume-ng-legacy-sources</module>
62
    <module>flume-ng-legacy-sources</module>
63
    <module>flume-ng-clients</module>
63
    <module>flume-ng-clients</module>
64
    <module>flume-ng-sdk</module>
64
    <module>flume-ng-sdk</module>
65
    <module>flume-ng-tests</module>
65
    <module>flume-ng-tests</module>

    
   
66
    <module>flume-tools</module>
66
  </modules>
67
  </modules>
67

    
   
68

   
68
  <profiles>
69
  <profiles>
69

    
   
70

   
70
    <profile>
71
    <profile>
[+20] [20] 815 lines
[+20]
886
        <version>1.4.0-SNAPSHOT</version>
887
        <version>1.4.0-SNAPSHOT</version>
887
      </dependency>
888
      </dependency>
888

    
   
889

   
889
      <dependency>
890
      <dependency>
890
        <groupId>org.apache.flume</groupId>
891
        <groupId>org.apache.flume</groupId>

    
   
892
        <artifactId>flume-tools</artifactId>

    
   
893
        <version>1.4.0-SNAPSHOT</version>

    
   
894
      </dependency>

    
   
895

   

    
   
896
      <dependency>

    
   
897
        <groupId>org.apache.flume</groupId>
891
        <artifactId>flume-ng-node</artifactId>
898
        <artifactId>flume-ng-node</artifactId>
892
        <version>1.4.0-SNAPSHOT</version>
899
        <version>1.4.0-SNAPSHOT</version>
893
      </dependency>
900
      </dependency>
894

    
   
901

   
895
      <dependency>
902
      <dependency>
[+20] [20] 181 lines
bin/flume-ng
Revision 22b95b8 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
New File
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
Revision cc0d38a New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
Revision 1918baa New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
Revision d3db896 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
Revision d9a2a9b New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
New File
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
Revision 4235a79 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
Revision fc47b23 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
Revision d6897e1 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
Revision 073042f New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
Revision 4860ac2 New Change
 
flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
Revision 1e668d2 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
Revision 0f7d14d New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
Revision 54978f8 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
Revision bef22ef New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
Revision f403422 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
Revision 563dbcc New Change
 
flume-ng-dist/pom.xml
Revision e266272 New Change
 
flume-ng-dist/src/main/assembly/bin.xml
Revision 74f0608 New Change
 
flume-ng-dist/src/main/assembly/src.xml
Revision 7fafab8 New Change
 
flume-tools/pom.xml
New File
 
flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
New File
 
flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
New File
 
flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
New File
 
flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
New File
 
flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
New File
 
  1. pom.xml: Loading...
  2. bin/flume-ng: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java: Loading...
  7. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  8. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java: Loading...
  9. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java: Loading...
  10. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java: Loading...
  11. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java: Loading...
  12. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java: Loading...
  13. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java: Loading...
  14. flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto: Loading...
  15. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java: Loading...
  16. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java: Loading...
  17. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java: Loading...
  18. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java: Loading...
  19. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java: Loading...
  20. flume-ng-dist/pom.xml: Loading...
  21. flume-ng-dist/src/main/assembly/bin.xml: Loading...
  22. flume-ng-dist/src/main/assembly/src.xml: Loading...
  23. flume-tools/pom.xml: Loading...
  24. flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java: Loading...
  25. flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java: Loading...
  26. flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java: Loading...
  27. flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java: Loading...
  28. flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java: Loading...