Review Board 1.7.22


1073: Move all journal stream management code into one place

Review Request #1688 - Created Aug. 31, 2011 and updated

Todd Lipcon
HDFS-2018
Reviewers
tlipcon
hadoop-common-git
Posting patch on review-board for easy review

 

Diff revision 1 (Latest)

  1. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java: Loading...
  2. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java: Loading...
  3. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java: Loading...
  4. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java: Loading...
  5. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java: Loading...
  6. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java: Loading...
  7. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java: Loading...
  8. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java: Loading...
  9. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java: Loading...
  10. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java: Loading...
  11. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java: Loading...
  12. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java: Loading...
  13. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java: Loading...
  14. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java: Loading...
  15. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java: Loading...
  16. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java: Loading...
  17. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java: Loading...
  18. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java: Loading...
  19. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java: Loading...
  20. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java: Loading...
  21. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java: Loading...
  22. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java: Loading...
  23. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java: Loading...
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
Revision d72509c New Change
[20] 18 lines
[+20]
19

    
   
19

   
20
import java.io.BufferedInputStream;
20
import java.io.BufferedInputStream;
21
import java.io.DataInputStream;
21
import java.io.DataInputStream;
22
import java.io.IOException;
22
import java.io.IOException;
23
import java.util.Iterator;
23
import java.util.Iterator;

    
   
24
import java.util.List;
24
import java.util.zip.Checksum;
25
import java.util.zip.Checksum;
25

    
   
26

   
26
import org.apache.hadoop.classification.InterfaceAudience;
27
import org.apache.hadoop.classification.InterfaceAudience;
27
import org.apache.hadoop.conf.Configuration;
28
import org.apache.hadoop.conf.Configuration;
28
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
29
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
29
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
30
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
30
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
31
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
31
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
32
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
32
import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.LogLoadPlan;

   
33
import org.apache.hadoop.io.IOUtils;
33
import org.apache.hadoop.io.IOUtils;
34
import org.apache.hadoop.util.StringUtils;
34
import org.apache.hadoop.util.StringUtils;
35

    
   
35

   
36
import com.google.common.base.Preconditions;
36
import com.google.common.base.Preconditions;

    
   
37
import com.google.common.collect.Lists;
37

    
   
38

   
38
/**
39
/**
39
 * Extension of FSImage for the backup node.
40
 * Extension of FSImage for the backup node.
40
 * This class handles the setup of the journaling 
41
 * This class handles the setup of the journaling 
41
 * spool on the backup namenode.
42
 * spool on the backup namenode.
[+20] [20] 217 lines
[+20] [+] private boolean tryConvergeJournalSpool() throws IOException {
259
          + lastAppliedTxId + " to " + target);
260
          + lastAppliedTxId + " to " + target);
260
      FSImageTransactionalStorageInspector inspector =
261
      FSImageTransactionalStorageInspector inspector =
261
        new FSImageTransactionalStorageInspector();
262
        new FSImageTransactionalStorageInspector();
262
      
263
      
263
      storage.inspectStorageDirs(inspector);
264
      storage.inspectStorageDirs(inspector);
264
      LogLoadPlan logLoadPlan = inspector.createLogLoadPlan(lastAppliedTxId,
265

   
265
          target - 1);
266
      editLog.recoverUnclosedStreams();
266
  
267
      Iterable<EditLogInputStream> editStreamsAll 
267
      logLoadPlan.doRecovery();
268
        = editLog.selectInputStreams(lastAppliedTxId, target - 1);
268
      loadEdits(logLoadPlan.getEditsFiles());
269
      // remove inprogress

    
   
270
      List<EditLogInputStream> editStreams = Lists.newArrayList();

    
   
271
      for (EditLogInputStream s : editStreamsAll) {

    
   
272
        if (s.getFirstTxId() != editLog.getCurSegmentTxId()) {

    
   
273
          editStreams.add(s);

    
   
274
        }

    
   
275
      }

    
   
276
      loadEdits(editStreams);
269
    }
277
    }
270
    
278
    
271
    // now, need to load the in-progress file
279
    // now, need to load the in-progress file
272
    synchronized (this) {
280
    synchronized (this) {
273
      if (lastAppliedTxId != editLog.getCurSegmentTxId() - 1) {
281
      if (lastAppliedTxId != editLog.getCurSegmentTxId() - 1) {
274
        LOG.debug("Logs rolled while catching up to current segment");
282
        LOG.debug("Logs rolled while catching up to current segment");
275
        return false; // drop lock and try again to load local logs
283
        return false; // drop lock and try again to load local logs
276
      }
284
      }
277
      
285
      
278
      EditLogInputStream stream = getEditLog().getInProgressFileInputStream();
286
      EditLogInputStream stream = null;

    
   
287
      Iterable<EditLogInputStream> editStreams

    
   
288
        = getEditLog().selectInputStreams(

    
   
289
            getEditLog().getCurSegmentTxId(),

    
   
290
            getEditLog().getCurSegmentTxId());

    
   
291
      for (EditLogInputStream s : editStreams) {

    
   
292
        if (s.getFirstTxId() == getEditLog().getCurSegmentTxId()) {

    
   
293
          stream = s;

    
   
294
        }

    
   
295
      }

    
   
296

   
279
      try {
297
      try {
280
        long remainingTxns = getEditLog().getLastWrittenTxId() - lastAppliedTxId;
298
        long remainingTxns = getEditLog().getLastWrittenTxId() - lastAppliedTxId;
281
        
299
        
282
        LOG.info("Going to finish converging with remaining " + remainingTxns
300
        LOG.info("Going to finish converging with remaining " + remainingTxns
283
            + " txns from in-progress stream " + stream);
301
            + " txns from in-progress stream " + stream);
[+20] [20] 111 lines
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
Revision 3cac667 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
Revision f754100 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
Revision 8921bc0 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
Revision 532b2f2 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
Revision 52a3dd4 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Revision 495c42e New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Revision db98569 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Revision 8b25901 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
Revision cec2eef New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java
Revision 65bfa0a New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
Revision 0814a14 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
Revision 991d7f5 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
Revision d62aaa7 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java
Revision 511e9c1 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
Revision 4a8edb8 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
Revision 4f698c0 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Revision 9e55946 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
Revision a673c5f New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
Revision a53a0bf New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
Revision 113dcbc New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
Revision 748caf4 New Change
 
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
Revision 31a7777 New Change
 
  1. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java: Loading...
  2. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java: Loading...
  3. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java: Loading...
  4. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java: Loading...
  5. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java: Loading...
  6. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java: Loading...
  7. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java: Loading...
  8. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java: Loading...
  9. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java: Loading...
  10. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java: Loading...
  11. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageStorageInspector.java: Loading...
  12. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java: Loading...
  13. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java: Loading...
  14. hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java: Loading...
  15. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java: Loading...
  16. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java: Loading...
  17. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java: Loading...
  18. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java: Loading...
  19. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java: Loading...
  20. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java: Loading...
  21. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java: Loading...
  22. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java: Loading...
  23. hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java: Loading...