Review Board 1.7.22


FLUME-1632: Persist progress on each file in file spooling client/source

Review Request #8596 - Created Dec. 14, 2012 and submitted

Mike Percy
FLUME-1632
Reviewers
Flume
flume-git
Defines EventDeserializer interface and uses it from the spooling source. Progress is persisted as bytes are read from the underlying file.
Unit tests pass.

Diff revision 1

This is not the most recent revision of the diff. The latest diff is revision 5. See what's changed.

1 2 3 4 5
1 2 3 4 5

  1. pom.xml: Loading...
  2. flume-ng-clients/pom.xml: Loading...
  3. flume-ng-clients/flume-ng-forwarder/pom.xml: Loading...
  4. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FileClient.java: Loading...
  5. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FlumeTool.java: Loading...
  6. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRegistry.java: Loading...
  7. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRunner.java: Loading...
  8. flume-ng-clients/flume-ng-forwarder/src/test/java/org/apache/flume/clients/forwarder/Blah2.java: Loading...
  9. flume-ng-clients/flume-ng-forwarder/src/test/resources/flume-log4jtest.properties: Loading...
  10. flume-ng-core/pom.xml: Loading...
  11. flume-ng-core/src/main/avro/TransferStateFileMeta.avsc: Loading...
  12. flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java: Loading...
  13. flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileEventParser.java: Loading...
  14. flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileLineReader.java: Loading...
  15. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java: Loading...
  16. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerFactory.java: Loading...
  17. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java: Loading...
  18. flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerDe.java: Loading...
  19. flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializer.java: Loading...
  20. flume-ng-core/src/main/java/org/apache/flume/serialization/LineDeserializer.java: Loading...
This diff has been split across 2 pages: 1 2 >
pom.xml
Revision 53ac96b New Change
[20] 509 lines
[+20]
510
          <groupId>org.apache.rat</groupId>
510
          <groupId>org.apache.rat</groupId>
511
          <artifactId>apache-rat-plugin</artifactId>
511
          <artifactId>apache-rat-plugin</artifactId>
512
          <version>0.7</version>
512
          <version>0.7</version>
513
          <executions>
513
          <executions>
514
            <execution>
514
            <execution>
515
              <id>test.rat</id>
515
              <id>verify.rat</id>
516
              <phase>test</phase>
516
              <phase>verify</phase>
517
              <goals>
517
              <goals>
518
                <goal>check</goal>
518
                <goal>check</goal>
519
              </goals>
519
              </goals>
520
              <configuration>
520
              <configuration>
521
                <excludes>
521
                <excludes>
[+20] [20] 5 lines
[+20]
527
                  <!-- ASF jenkins box puts the Maven repo in our root directory. -->
527
                  <!-- ASF jenkins box puts the Maven repo in our root directory. -->
528
                  <exclude>.repository/</exclude>
528
                  <exclude>.repository/</exclude>
529
                  <exclude>**/*.diff</exclude>
529
                  <exclude>**/*.diff</exclude>
530
                  <exclude>**/*.patch</exclude>
530
                  <exclude>**/*.patch</exclude>
531
                  <exclude>**/*.avsc</exclude>
531
                  <exclude>**/*.avsc</exclude>

    
   
532
                  <exclude>**/*.avro</exclude>
532
                  <exclude>**/docs/**</exclude>
533
                  <exclude>**/docs/**</exclude>
533
                </excludes>
534
                </excludes>
534
              </configuration>
535
              </configuration>
535
            </execution>
536
            </execution>
536
          </executions>
537
          </executions>
[+20] [20] 404 lines
[+20]
941
        <artifactId>flume-ng-log4jappender</artifactId>
942
        <artifactId>flume-ng-log4jappender</artifactId>
942
        <version>1.4.0-SNAPSHOT</version>
943
        <version>1.4.0-SNAPSHOT</version>
943
      </dependency>
944
      </dependency>
944

    
   
945

   
945
      <dependency>
946
      <dependency>

    
   
947
        <groupId>org.apache.flume.flume-ng-clients</groupId>

    
   
948
        <artifactId>flume-ng-forwarder</artifactId>

    
   
949
        <version>1.3.0-SNAPSHOT</version>

    
   
950
      </dependency>

    
   
951

   

    
   
952
      <dependency>
946
        <groupId>com.sun.jersey</groupId>
953
        <groupId>com.sun.jersey</groupId>
947
        <artifactId>jersey-core</artifactId>
954
        <artifactId>jersey-core</artifactId>
948
        <version>1.8</version>
955
        <version>1.8</version>
949
      </dependency>
956
      </dependency>
950

    
   
957

   
[+20] [20] 75 lines
flume-ng-clients/pom.xml
Revision 7ea2d1d New Change
 
flume-ng-clients/flume-ng-forwarder/pom.xml
New File
 
flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FileClient.java
New File
 
flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FlumeTool.java
New File
 
flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRegistry.java
New File
 
flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRunner.java
New File
 
flume-ng-clients/flume-ng-forwarder/src/test/java/org/apache/flume/clients/forwarder/Blah2.java
New File
 
flume-ng-clients/flume-ng-forwarder/src/test/resources/flume-log4jtest.properties
New File
 
flume-ng-core/pom.xml
Revision 2554a0e New Change
 
flume-ng-core/src/main/avro/TransferStateFileMeta.avsc
New File
 
flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java
Revision 37e9ffa New Change
 
flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileEventParser.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileLineReader.java
Revision 8362299 New Change
 
flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerFactory.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerDe.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializer.java
Revision a418935 New Change
 
flume-ng-core/src/main/java/org/apache/flume/serialization/LineDeserializer.java
New File
 
  1. pom.xml: Loading...
  2. flume-ng-clients/pom.xml: Loading...
  3. flume-ng-clients/flume-ng-forwarder/pom.xml: Loading...
  4. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FileClient.java: Loading...
  5. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/FlumeTool.java: Loading...
  6. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRegistry.java: Loading...
  7. flume-ng-clients/flume-ng-forwarder/src/main/java/org/apache/flume/clients/forwarder/ToolRunner.java: Loading...
  8. flume-ng-clients/flume-ng-forwarder/src/test/java/org/apache/flume/clients/forwarder/Blah2.java: Loading...
  9. flume-ng-clients/flume-ng-forwarder/src/test/resources/flume-log4jtest.properties: Loading...
  10. flume-ng-core/pom.xml: Loading...
  11. flume-ng-core/src/main/avro/TransferStateFileMeta.avsc: Loading...
  12. flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java: Loading...
  13. flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileEventParser.java: Loading...
  14. flume-ng-core/src/main/java/org/apache/flume/client/avro/SpoolingFileLineReader.java: Loading...
  15. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializer.java: Loading...
  16. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerFactory.java: Loading...
  17. flume-ng-core/src/main/java/org/apache/flume/serialization/EventDeserializerType.java: Loading...
  18. flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerDe.java: Loading...
  19. flume-ng-core/src/main/java/org/apache/flume/serialization/EventSerializer.java: Loading...
  20. flume-ng-core/src/main/java/org/apache/flume/serialization/LineDeserializer.java: Loading...
This diff has been split across 2 pages: 1 2 >