Feat: Added an option to fast forward the ledger state

src/main/java/com/iota/iri/service/ledger/LedgerService.java

@@ -14,6 +14,20 @@
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface LedgerService {
+    /**
+     * Restores the ledger state after a restart of IRI, which allows us to fast forward to the point where we
+     * stopped before the restart.<br />
+     * <br />
+     * It looks for the last solid milestone that was applied to the ledger in the database and then replays all
+     * milestones leading up to this point by applying them to the latest snapshot. We do not check every single
+     * milestone again but assume that the data in the database is correct. If the database would have any
+     * inconsistencies and the application fails, the latest solid milestone tracker will check and apply the milestones
+     * one by one and repair the corresponding inconsistencies.<br />
+     *
+     * @throws LedgerException if anything unexpected happens while trying to restore the ledger state
+     */
+    void restoreLedgerState() throws LedgerException;
+
     /**
      * Applies the given milestone to the ledger state.<br />
      * <br />

src/main/java/com/iota/iri/service/ledger/impl/LedgerServiceImpl.java

@@ -71,6 +71,21 @@ public LedgerServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider,
         return this;
     }
 
+    @Override
+    public void restoreLedgerState() throws LedgerException {
+        try {
+            milestoneService.findLatestProcessedSolidMilestoneInDatabase().ifPresent(milestoneViewModel -> {
+                try {
+                    snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestoneViewModel.index());
+                } catch (SnapshotException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        } catch (Exception e) {
+            throw new LedgerException("unexpected error while restoring the ledger state", e);
+        }
+    }
+
     @Override
     public boolean applyMilestoneToLedger(MilestoneViewModel milestone) throws LedgerException {
         if(generateStateDiff(milestone)) {

src/main/java/com/iota/iri/service/milestone/MilestoneService.java

@@ -1,15 +1,31 @@
 package com.iota.iri.service.milestone;
 
+import com.iota.iri.controllers.MilestoneViewModel;
 import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.crypto.SpongeFactory;
 import com.iota.iri.model.Hash;
 
+import java.util.Optional;
+
 /**
  * Represents the service that contains all the relevant business logic for interacting with milestones.<br />
  * <br />
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface MilestoneService {
+    /**
+     * This method tries to find the latest solid milestone that was previously processed by IRI (before a restart) by
+     * performing a search in the database.<br />
+     * <br />
+     * It determines if the milestones were processed by checking the {@code snapshotIndex} value of their corresponding
+     * transactions.<br />
+     *
+     * @return the latest solid milestone that was previously processed by IRI or an empty value if no previously
+     *         processed solid milestone can be found
+     * @throws MilestoneException if anything unexpected happend while performing the search
+     */
+    Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException;
+
     /**
      * Analyzes the given transaction to determine if it is a valid milestone.<br />
      * <br />

src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java

@@ -76,6 +76,11 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac
     private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService(
             "Latest Solid Milestone Tracker", log.delegate());
 
+    /**
+     * Boolean flag that is used to identify the first iteration of the background worker.<br />
+     */
+    private boolean firstRun = true;
+
     /**
      * Holds the milestone index of the milestone that caused the repair logic to get started.<br />
      */
@@ -140,22 +145,32 @@ public void shutdown() {
     @Override
     public void checkForNewLatestSolidMilestones() throws MilestoneException {
         try {
+            if (firstRun) {
+                firstRun = false;
+
+                ledgerService.restoreLedgerState();
+                logChange(snapshotProvider.getInitialSnapshot().getIndex());
+            }
+
             int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
             if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) {
                 MilestoneViewModel nextMilestone;
                 while (!Thread.currentThread().isInterrupted() &&
                         (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null &&
                         TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) {
 
-                    syncLatestMilestoneTracker(nextMilestone);
+                    syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index());
                     applySolidMilestoneToLedger(nextMilestone);
                     logChange(currentSolidMilestoneIndex);
 
                     currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
                 }
+            } else {
+                syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(),
+                        snapshotProvider.getLatestSnapshot().getIndex());
             }
         } catch (Exception e) {
-            throw new MilestoneException(e);
+            throw new MilestoneException("unexpected error while checking for new latest solid milestones", e);
         }
     }
 
@@ -237,11 +252,12 @@ private void stopRepair() {
      * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone
      *       index.
      *
-     * @param processedMilestone the milestone that currently gets processed
+     * @param milestoneHash transaction hash of the milestone
+     * @param milestoneIndex milestone index
      */
-    private void syncLatestMilestoneTracker(MilestoneViewModel processedMilestone) {
-        if(processedMilestone.index() > latestMilestoneTracker.getLatestMilestoneIndex()) {
-            latestMilestoneTracker.setLatestMilestone(processedMilestone.getHash(), processedMilestone.index());
+    private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) {
+        if(milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) {
+            latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex);
         }
     }
 

src/main/java/com/iota/iri/service/milestone/impl/MilestoneServiceImpl.java

@@ -101,6 +101,63 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide
 
     //region {PUBLIC METHODS] //////////////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * {@inheritDoc}
+     * <br />
+     * We first check the trivial case where the node was fully synced. If no processed solid milestone could be found
+     * within the last two milestones of the node, we perform a binary search from present to past, which reduces the
+     * amount of database requests to a minimum (even with a huge amount of milestones in the database).<br />
+     */
+    @Override
+    public Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException {
+        try {
+            // if we have no milestone in our database -> abort
+            MilestoneViewModel latestMilestone = MilestoneViewModel.latest(tangle);
+            if (latestMilestone == null) {
+                return Optional.empty();
+            }
+
+             // trivial case #1: the node was fully synced
+            if (wasMilestoneAppliedToLedger(latestMilestone)) {
+                return Optional.of(latestMilestone);
+            }
+
+             // trivial case #2: the node was fully synced but the last milestone was not processed, yet
+            MilestoneViewModel latestMilestonePredecessor = MilestoneViewModel.findClosestPrevMilestone(tangle,
+                    latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex());
+            if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) {
+                return Optional.of(latestMilestonePredecessor);
+            }
+
+             // non-trivial case: do a binary search from the end of the ledger to the front
+            Optional<MilestoneViewModel> lastAppliedCandidate = Optional.empty();
+            int rangeEnd = latestMilestone.index();
+            int rangeStart = snapshotProvider.getInitialSnapshot().getIndex() + 1;
+            while (rangeEnd - rangeStart >= 0) {
+                // if no candidate found in range -> return last candidate
+                MilestoneViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd);
+                if (currentCandidate == null) {
+                    return lastAppliedCandidate;
+                }
+
+                 // if the milestone was applied -> continue to search for later ones that were also applied
+                if (wasMilestoneAppliedToLedger(currentCandidate)) {
+                    rangeStart = currentCandidate.index() + 1;
+                     lastAppliedCandidate = Optional.of(currentCandidate);
+                }
+
+                 // if the milestone was not applied -> continue to search for earlier ones
+                else {
+                    rangeEnd = currentCandidate.index() - 1;
+                }
+            }
+            return lastAppliedCandidate;
+        } catch (Exception e) {
+            throw new MilestoneException(
+                    "unexpected error while trying to find the latest processed solid milestone in the database", e);
+        }
+    }
+
     @Override
     public void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException {
         if (index <= 0) {
@@ -224,6 +281,51 @@ public boolean isTransactionConfirmed(TransactionViewModel transaction) {
 
     //region [PRIVATE UTILITY METHODS] /////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * Determines the milestone in the middle of the range defined by {@code rangeStart} and {@code rangeEnd}.<br />
+     * <br />
+     * It is used by the binary search algorithm of {@link #findLatestProcessedSolidMilestoneInDatabase()}. It first
+     * calculates the index that represents the middle of the range and then tries to find the milestone that is closest
+     * to this index.<br/>
+     * <br />
+     * Note: We start looking for younger milestones first, because most of the times the latest processed solid
+     *       milestone is close to the end.<br />
+     *
+     * @param rangeStart the milestone index representing the start of our search range
+     * @param rangeEnd the milestone index representing the end of our search range
+     * @return the milestone that is closest to the middle of the given range or {@code null} if no milestone can be
+     *         found
+     * @throws Exception if anything unexpected happens while trying to get the milestone
+     */
+    private MilestoneViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception {
+        int range = rangeEnd - rangeStart;
+        int middleOfRange = rangeEnd - range / 2;
+
+        MilestoneViewModel milestone = MilestoneViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd);
+        if (milestone == null) {
+            milestone = MilestoneViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart);
+        }
+
+        return milestone;
+    }
+
+    /**
+     * Checks if the milestone was applied to the ledger at some point in the past (before a restart of IRI).<br />
+     * <br />
+     * Since the {@code snapshotIndex} value is used as a flag to determine if the milestone was already applied to the
+     * ledger, we can use it to determine if it was processed by IRI in the past. If this value is set we should also
+     * have a corresponding {@link StateDiff} entry in the database.<br />
+     *
+     * @param milestone the milestone that shall be checked
+     * @return {@code true} if the milestone has been processed by IRI before and {@code false} otherwise
+     * @throws Exception if anything unexpected happens while checking the milestone
+     */
+    private boolean wasMilestoneAppliedToLedger(MilestoneViewModel milestone) throws Exception {
+        TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash());
+        return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT &&
+                milestoneTransaction.snapshotIndex() != 0;
+    }
+
     /**
      * This method implements the logic described by {@link #updateMilestoneIndexOfMilestoneTransactions(Hash, int)} but
      * accepts some additional parameters that allow it to be reused by different parts of this service.<br />