Review Board 1.7.22


FLUME-1731: SpoolableDirectorySource should have configurable support for deleting files it has already completed instead of renaming

Review Request #8957 - Created Jan. 15, 2013 and submitted

Mike Percy
FLUME-1731
Reviewers
Flume
flume-git
Implement delete support for SpoolableDirectorySource. Also refactored the constructor logic to improve maintainability by putting in a Builder for the ReliableSpoolingFileEventReader.
Added unit test. Existing unit tests pass.
flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java
Revision da23a75 New Change
[20] 33 lines
[+20]
34
import org.apache.commons.cli.CommandLineParser;
34
import org.apache.commons.cli.CommandLineParser;
35
import org.apache.commons.cli.GnuParser;
35
import org.apache.commons.cli.GnuParser;
36
import org.apache.commons.cli.HelpFormatter;
36
import org.apache.commons.cli.HelpFormatter;
37
import org.apache.commons.cli.Options;
37
import org.apache.commons.cli.Options;
38
import org.apache.commons.cli.ParseException;
38
import org.apache.commons.cli.ParseException;
39
import org.apache.flume.Context;

   
40
import org.apache.flume.Event;
39
import org.apache.flume.Event;
41
import org.apache.flume.EventDeliveryException;
40
import org.apache.flume.EventDeliveryException;
42
import org.apache.flume.FlumeException;
41
import org.apache.flume.FlumeException;
43
import org.apache.flume.annotations.InterfaceAudience;
42
import org.apache.flume.annotations.InterfaceAudience;
44
import org.apache.flume.annotations.InterfaceStability;
43
import org.apache.flume.annotations.InterfaceStability;
[+20] [20] 157 lines
[+20] [+] private void run() throws IOException, FlumeException,
202

    
   
201

   
203
    try {
202
    try {
204
      if (fileName != null) {
203
      if (fileName != null) {
205
        reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
204
        reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
206
      } else if (dirName != null) {
205
      } else if (dirName != null) {
207
        reader = new ReliableSpoolingFileEventReader(
206
        reader = new ReliableSpoolingFileEventReader.Builder()
208
            new File(dirName), ".COMPLETED",
207
            .spoolDirectory(new File(dirName)).build();
209
            "^$", new File(new File(dirName), ".flumespool"),

   
210
            false, "",

   
211
            "LINE", new Context());

   
212
      } else {
208
      } else {
213
        reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
209
        reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
214
      }
210
      }
215

    
   
211

   
216
      long lastCheck = System.currentTimeMillis();
212
      long lastCheck = System.currentTimeMillis();
[+20] [20] 33 lines
flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
Revision f2d587f New Change
 
flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java
Revision 167193c New Change
 
flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java
Revision 67549e0 New Change
 
flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java
Revision bc10243 New Change
 
flume-ng-core/src/test/java/org/apache/flume/serialization/TestReliableSpoolingFileEventReader.java
Revision abc1827 New Change
 
  1. flume-ng-core/src/main/java/org/apache/flume/client/avro/AvroCLIClient.java: Loading...
  2. flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java: Loading...
  3. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java: Loading...
  4. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java: Loading...
  5. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java: Loading...
  6. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java: Loading...
  7. flume-ng-core/src/test/java/org/apache/flume/serialization/TestReliableSpoolingFileEventReader.java: Loading...