Review Board 1.7.22


FLUME-1761: FileChannel can NPE when log metadata file is empty

Review Request #8414 - Created Dec. 7, 2012 and updated

Brock Noland
trunk
FLUME-1761
Reviewers
Flume
flume-git
1) Adds better error messages if for any reason Commit,Take,Put,Rollback encountered an null pointer. Shouldn't happen.
2) Fast replay handles the case where we have an empty log or metadata file.
3) LogFileFactory throws an informative error message when the meta file is empty but the log not.
4) LogFileFactory throws an EOF when log file and meta are empty sot he log can be ignored during replay.
Added tests which test 2, 3, 4. The behavior is the same for 1 we just have a better error message.
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CheckpointRebuilder.java
Revision 748f49a New Change
[20] 20 lines
[+20]
21
import com.google.common.base.Preconditions;
21
import com.google.common.base.Preconditions;
22
import com.google.common.collect.HashMultimap;
22
import com.google.common.collect.HashMultimap;
23
import com.google.common.collect.Lists;
23
import com.google.common.collect.Lists;
24
import com.google.common.collect.SetMultimap;
24
import com.google.common.collect.SetMultimap;
25
import com.google.common.collect.Sets;
25
import com.google.common.collect.Sets;

    
   
26

   

    
   
27
import java.io.EOFException;
26
import java.io.File;
28
import java.io.File;
27
import java.io.IOException;
29
import java.io.IOException;
28
import java.util.Arrays;
30
import java.util.Arrays;
29
import java.util.List;
31
import java.util.List;
30
import java.util.Set;
32
import java.util.Set;
[+20] [20] 29 lines
[+20] [+] public class CheckpointRebuilder {
60

    
   
62

   
61
  public boolean rebuild() throws IOException, Exception {
63
  public boolean rebuild() throws IOException, Exception {
62
    LOG.info("Attempting to fast replay the log files.");
64
    LOG.info("Attempting to fast replay the log files.");
63
    List<LogFile.SequentialReader> logReaders = Lists.newArrayList();
65
    List<LogFile.SequentialReader> logReaders = Lists.newArrayList();
64
    for (File logFile : logFiles) {
66
    for (File logFile : logFiles) {

    
   
67
      try {
65
      logReaders.add(LogFileFactory.getSequentialReader(logFile, null));
68
        logReaders.add(LogFileFactory.getSequentialReader(logFile, null));

    
   
69
      } catch(EOFException e) {

    
   
70
        LOG.warn("Ignoring " + logFile + " due to EOF", e);

    
   
71
      }
66
    }
72
    }
67
    long transactionIDSeed = 0;
73
    long transactionIDSeed = 0;
68
    long writeOrderIDSeed = 0;
74
    long writeOrderIDSeed = 0;
69
    try {
75
    try {
70
      for (LogFile.SequentialReader log : logReaders) {
76
      for (LogFile.SequentialReader log : logReaders) {
[+20] [20] 188 lines
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Commit.java
Revision 62f4451 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileFactory.java
Revision 4783448 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
Revision b4c197e New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
Revision d47b1c8 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Rollback.java
Revision cc9ce86 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Take.java
Revision e61bf7e New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
Revision 70098a0 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
Revision a165d6a New Change
 
  1. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CheckpointRebuilder.java: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Commit.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileFactory.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Rollback.java: Loading...
  7. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Take.java: Loading...
  8. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java: Loading...
  9. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java: Loading...