platform/vcs-log/impl/src/com/intellij/vcs/log/data/VcsLogRefresherImpl.java

/*
 * Copyright 2000-2014 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.vcs.log.data;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.Task;
import com.intellij.openapi.progress.impl.ProgressManagerImpl;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.vcs.VcsException;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Consumer;
import com.intellij.util.Function;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.ui.UIUtil;
import com.intellij.vcs.log.*;
import com.intellij.vcs.log.graph.GraphCommit;
import com.intellij.vcs.log.graph.GraphCommitImpl;
import com.intellij.vcs.log.graph.PermanentGraph;
import com.intellij.vcs.log.impl.RequirementsImpl;
import com.intellij.vcs.log.util.StopWatch;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;

public class VcsLogRefresherImpl implements VcsLogRefresher {
  
  private static final Logger LOG = Logger.getInstance(VcsLogRefresherImpl.class);

  @NotNull private final Project myProject;
  @NotNull private final VcsLogHashMap myHashMap;
  @NotNull private final Map<VirtualFile, VcsLogProvider> myProviders;
  @NotNull private final VcsUserRegistry myUserRegistry;
  @NotNull private final Map<Hash, VcsCommitMetadata> myTopCommitsDetailsCache;
  @NotNull private final Consumer<Exception> myExceptionHandler;
  private final int myRecentCommitCount;

  @NotNull private final SingleTaskController<RefreshRequest, DataPack> mySingleTaskController;

  @NotNull private DataPack myDataPack = EmptyDataPack.getInstance();

  public VcsLogRefresherImpl(@NotNull final Project project,
                             @NotNull VcsLogHashMap hashMap,
                             @NotNull Map<VirtualFile, VcsLogProvider> providers,
                             @NotNull final VcsUserRegistry userRegistry,
                             @NotNull Map<Hash, VcsCommitMetadata> topCommitsDetailsCache,
                             @NotNull final Consumer<DataPack> dataPackUpdateHandler,
                             @NotNull Consumer<Exception> exceptionHandler, int recentCommitsCount) {
    myProject = project;
    myHashMap = hashMap;
    myProviders = providers;
    myUserRegistry = userRegistry;
    myTopCommitsDetailsCache = topCommitsDetailsCache;
    myExceptionHandler = exceptionHandler;
    myRecentCommitCount = recentCommitsCount;

    Consumer<DataPack> dataPackUpdater = new Consumer<DataPack>() {
      @Override
      public void consume(@NotNull DataPack dataPack) {
        myDataPack = dataPack;
        dataPackUpdateHandler.consume(dataPack);
      }
    };
    mySingleTaskController = new SingleTaskController<RefreshRequest, DataPack>(dataPackUpdater) {
      @Override
      protected void startNewBackgroundTask() {
        UIUtil.invokeLaterIfNeeded(new Runnable() {
          @Override
          public void run() {
            ProgressManagerImpl.runProcessWithProgressAsynchronously(new MyRefreshTask(myDataPack));
          }
        });
      }
    };
  }

  @NotNull
  @Override
  public DataPack readFirstBlock() {
    try {
      Map<VirtualFile, Collection<VcsRef>> refs = loadRefsFromVcs(myProviders);
      Set<VirtualFile> roots = myProviders.keySet();
      Map<VirtualFile, VcsLogProvider.Requirements> requirements = prepareSimpleRequirements(roots, myRecentCommitCount);
      Map<VirtualFile, List<? extends TimedVcsCommit>> commits = loadRecentCommitsFromVcs(myProviders, requirements,
                                                                                          myUserRegistry, myTopCommitsDetailsCache);
      List<? extends TimedVcsCommit> compoundLog = compound(commits.values());
      List<GraphCommit<Integer>> compactedLog = compactCommits(compoundLog, myHashMap);
      DataPack dataPack = DataPack.build(compactedLog, new RefsModel(refs, myHashMap.asIndexGetter()),
                                         myHashMap.asIndexGetter(), myHashMap.asHashGetter(), myProviders, false);
      mySingleTaskController.request(RefreshRequest.RELOAD_ALL); // build/rebuild the full log in bg
      return dataPack;
    }
    catch (VcsException e) {
      myExceptionHandler.consume(e);
      return EmptyDataPack.getInstance();
    }
  }

  @Override
  public void refresh(@NotNull Collection<VirtualFile> rootsToRefresh) {
    if (!rootsToRefresh.isEmpty()) {
      mySingleTaskController.request(new RefreshRequest(rootsToRefresh));
    }
  }

  @NotNull
  private static Map<VirtualFile, VcsLogProvider.Requirements> prepareSimpleRequirements(@NotNull Collection<VirtualFile> roots,
                                                                                         final int commitCount) {
    final VcsLogProvider.Requirements requirements = new VcsLogProvider.Requirements() {
      @Override
      public int getCommitCount() {
        return commitCount;
      }
    };
    return ContainerUtil.map2Map(roots, new Function<VirtualFile, Pair<VirtualFile, VcsLogProvider.Requirements>>() {
      @Override
      public Pair<VirtualFile, VcsLogProvider.Requirements> fun(VirtualFile file) {
        return Pair.create(file, requirements);
      }
    });
  }

  @NotNull
  private static Map<VirtualFile, Collection<VcsRef>> loadRefsFromVcs(@NotNull Map<VirtualFile, VcsLogProvider> providers)
    throws VcsException {
    final StopWatch sw = StopWatch.start("loading refs");
    final Map<VirtualFile, Collection<VcsRef>> refs = ContainerUtil.newHashMap();
    new ProviderIterator() {
      @Override
      void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
        refs.put(root, provider.readAllRefs(root));
        sw.rootCompleted(root);
      }
    }.iterate(providers);
    sw.report();
    return refs;
  }

  @NotNull
  private static Map<VirtualFile, List<? extends TimedVcsCommit>> loadRecentCommitsFromVcs(@NotNull Map<VirtualFile, VcsLogProvider> providers,
                                                                                           @NotNull final Map<VirtualFile, VcsLogProvider.Requirements> requirements,
                                                                                           @NotNull final VcsUserRegistry userRegistry,
                                                                                           @NotNull final Map<Hash, VcsCommitMetadata> topCommitsDetailsCache)
    throws VcsException {
    final StopWatch sw = StopWatch.start("loading commits");
    final Map<VirtualFile, List<? extends TimedVcsCommit>> commits = ContainerUtil.newHashMap();
    new ProviderIterator() {
      @Override
      public void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
        List<? extends VcsCommitMetadata> metadatas = provider.readFirstBlock(root, requirements.get(root));
        storeUsersAndDetails(metadatas, userRegistry, topCommitsDetailsCache);
        commits.put(root, metadatas);
        sw.rootCompleted(root);
      }
    }.iterate(providers);
    sw.report();
    return commits;
  }

  /**
   * Compounds logs from different repositories into a single multi-repository log.
   */
  @NotNull
  private static List<? extends TimedVcsCommit> compound(@NotNull Collection<List<? extends TimedVcsCommit>> commits) {
    StopWatch sw = StopWatch.start("multi-repo join");
    List<? extends TimedVcsCommit> joined = new VcsLogMultiRepoJoiner().join(commits);
    sw.report();
    return joined;
  }

  @NotNull
  private static List<GraphCommit<Integer>> compactCommits(@NotNull List<? extends TimedVcsCommit> commits,
                                                           @NotNull final VcsLogHashMap hashMap) {
    StopWatch sw = StopWatch.start("compacting commits");
    List<GraphCommit<Integer>> map = ContainerUtil.map(commits, new Function<TimedVcsCommit, GraphCommit<Integer>>() {
      @NotNull
      @Override
      public GraphCommit<Integer> fun(@NotNull TimedVcsCommit commit) {
        return new GraphCommitImpl<Integer>(hashMap.getCommitIndex(commit.getId()),
                                            ContainerUtil.map(commit.getParents(), hashMap.asIndexGetter()), commit.getTimestamp());
      }
    });
    hashMap.flush();
    sw.report();
    return map;
  }

  private static void storeUsersAndDetails(@NotNull List<? extends VcsCommitMetadata> metadatas, @NotNull VcsUserRegistry userRegistry,
                                           @NotNull Map<Hash, VcsCommitMetadata> topCommitsDetailsCache) {
    for (VcsCommitMetadata detail : metadatas) {
      userRegistry.addUser(detail.getAuthor());
      userRegistry.addUser(detail.getCommitter());
      topCommitsDetailsCache.put(detail.getId(), detail);
    }
  }

  private class MyRefreshTask extends Task.Backgroundable {

    @NotNull private DataPack myCurrentDataPack;

    // collects loaded info from different roots, which refresh was requested consecutively within a single task
    private final Map<VirtualFile, LogAndRefs> myLoadedInfos = ContainerUtil.newHashMap();

    private class LogAndRefs {
      List<? extends TimedVcsCommit> log;
      Collection<VcsRef> refs;
      LogAndRefs(Collection<VcsRef> refs, List<? extends TimedVcsCommit> commits) {
        this.refs = refs;
        this.log = commits;
      }
    }

    MyRefreshTask(@NotNull DataPack currentDataPack) {
      super(VcsLogRefresherImpl.this.myProject, "Refreshing history...", false);
      myCurrentDataPack = currentDataPack;
    }

    @Override
    public void run(@NotNull ProgressIndicator indicator) {
      DataPack dataPack = myCurrentDataPack;
      while (true) {
        List<RefreshRequest> requests = mySingleTaskController.popRequests();
        Collection<VirtualFile> rootsToRefresh = getRootsToRefresh(requests);
        if (rootsToRefresh.isEmpty()) {
          mySingleTaskController.taskCompleted(dataPack);
          break;
        }
        dataPack = doRefresh(rootsToRefresh);
      }
    }

    @NotNull
    private Collection<VirtualFile> getRootsToRefresh(@NotNull List<RefreshRequest> requests) {
      Collection<VirtualFile> rootsToRefresh = ContainerUtil.newArrayList();
      for (RefreshRequest request : requests) {
        if (request == RefreshRequest.RELOAD_ALL) {
          myCurrentDataPack = EmptyDataPack.getInstance();
          return myProviders.keySet();
        }
        rootsToRefresh.addAll(request.rootsToRefresh);
      }
      return rootsToRefresh;
    }

    @NotNull
    private DataPack doRefresh(@NotNull Collection<VirtualFile> roots) {
      StopWatch sw = StopWatch.start("refresh");
      PermanentGraph<Integer> permanentGraph = myCurrentDataPack.isFull() ? myCurrentDataPack.getPermanentGraph() : null;
      Map<VirtualFile, Collection<VcsRef>> currentRefs = myCurrentDataPack.getRefsModel().getAllRefsByRoot();
      try {
        if (permanentGraph != null) {
          loadLogAndRefs(roots, currentRefs, myRecentCommitCount);
          List<? extends TimedVcsCommit> compoundLog = compound(ContainerUtil.map(myLoadedInfos.values(),
            new Function<LogAndRefs, List<? extends TimedVcsCommit>>() {
              @Override
              public List<? extends TimedVcsCommit> fun(LogAndRefs refs) {
                return refs.log;
              }
            }));
          List<GraphCommit<Integer>> preparedLog = compactCommits(compoundLog, myHashMap);
          Map<VirtualFile, Collection<VcsRef>> allNewRefs = getAllNewRefs(myLoadedInfos, currentRefs);
          List<GraphCommit<Integer>> joinedFullLog = join(preparedLog, permanentGraph.getAllCommits(), currentRefs, allNewRefs);
          if (joinedFullLog != null) {
            return DataPack.build(joinedFullLog, new RefsModel(allNewRefs, myHashMap.asIndexGetter()),
                                  myHashMap.asIndexGetter(), myHashMap.asHashGetter(), myProviders, true);
          }
          // couldn't join => need to reload everything; this shouldn't happen often, the error is logged in join().
        }

        Pair<PermanentGraph<Integer>, Map<VirtualFile, Collection<VcsRef>>> fullLogAndRefs = loadFullLog();
        return DataPack.build(fullLogAndRefs.first, myProviders, new RefsModel(fullLogAndRefs.second, myHashMap.asIndexGetter()), true);
      }
      catch (Exception e) {
        myExceptionHandler.consume(e);
        return EmptyDataPack.getInstance();
      }
      finally {
        sw.report();
      }
    }

    @NotNull
    private Map<VirtualFile, Collection<VcsRef>> getAllNewRefs(@NotNull Map<VirtualFile, LogAndRefs> newInfo,
                                                               @NotNull Map<VirtualFile, Collection<VcsRef>> previousRefs) {
      Map<VirtualFile, Collection<VcsRef>> result = ContainerUtil.newHashMap();
      for (VirtualFile root : previousRefs.keySet()) {
        result.put(root, newInfo.containsKey(root) ? newInfo.get(root).refs : previousRefs.get(root));
      }
      return result;
    }

    private void loadLogAndRefs(@NotNull Collection<VirtualFile> roots, @NotNull Map<VirtualFile, Collection<VcsRef>> prevRefs,
                                int commitCount) throws VcsException {
      Map<VirtualFile, VcsLogProvider> providers = getProviders(roots);
      Map<VirtualFile, Collection<VcsRef>> refs = loadRefsFromVcs(providers);
      Map<VirtualFile, VcsLogProvider.Requirements> requirements = prepareRequirements(roots, commitCount, prevRefs, refs);
      Map<VirtualFile, List<? extends TimedVcsCommit>> commits = loadRecentCommitsFromVcs(providers, requirements,
                                                                                          myUserRegistry, myTopCommitsDetailsCache);
      for (VirtualFile root : roots) {
        myLoadedInfos.put(root, new LogAndRefs(refs.get(root), commits.get(root)));
      }
    }

    @NotNull
    private Map<VirtualFile, VcsLogProvider.Requirements> prepareRequirements(@NotNull Collection<VirtualFile> roots,
                                                                              int commitCount,
                                                                              @NotNull Map<VirtualFile, Collection<VcsRef>> prevRefs,
                                                                              @NotNull Map<VirtualFile, Collection<VcsRef>> newRefs) {
      Map<VirtualFile, VcsLogProvider.Requirements> requirements = ContainerUtil.newHashMap();
      for (VirtualFile root : roots) {
        requirements.put(root, new RequirementsImpl(commitCount, true, getRefsForRoot(prevRefs, root), getRefsForRoot(newRefs, root)));
      }
      return requirements;
    }

    @NotNull
    private Set<VcsRef> getRefsForRoot(@NotNull Map<VirtualFile, Collection<VcsRef>> map, @NotNull VirtualFile root) {
      Collection<VcsRef> refs = map.get(root);
      return refs == null ? Collections.<VcsRef>emptySet() : new HashSet<VcsRef>(refs);
    }

    @NotNull
    private Map<VirtualFile, VcsLogProvider> getProviders(@NotNull Collection<VirtualFile> roots) {
      Map<VirtualFile, VcsLogProvider> providers = ContainerUtil.newHashMap();
      for (VirtualFile root : roots) {
        providers.put(root, myProviders.get(root));
      }
      return providers;
    }

    @Nullable
    private List<GraphCommit<Integer>> join(@NotNull List<GraphCommit<Integer>> recentCommits, @NotNull List<GraphCommit<Integer>> fullLog,
                                            @NotNull Map<VirtualFile, Collection<VcsRef>> previousRefs,
                                            @NotNull Map<VirtualFile, Collection<VcsRef>> newRefs) {
      StopWatch sw = StopWatch.start("joining new commits");
      Function<VcsRef, Integer> ref2Int = new Function<VcsRef, Integer>() {
        @NotNull
        @Override
        public Integer fun(@NotNull VcsRef ref) {
          return myHashMap.getCommitIndex(ref.getCommitHash());
        }
      };
      Collection<Integer> prevRefIndices = ContainerUtil.map(ContainerUtil.concat(previousRefs.values()), ref2Int);
      Collection<Integer> newRefIndices = ContainerUtil.map(ContainerUtil.concat(newRefs.values()), ref2Int);
      try {
        List<GraphCommit<Integer>> commits = new VcsLogJoiner<Integer, GraphCommit<Integer>>().addCommits(fullLog, prevRefIndices,
                                                                                                          recentCommits,
                                                                                                          newRefIndices).first;
        sw.report();
        return commits;
      }
      catch (IllegalStateException e) {
        LOG.info(e);
        return null;
      }
    }

    @NotNull
    private Pair<PermanentGraph<Integer>, Map<VirtualFile, Collection<VcsRef>>> loadFullLog() throws VcsException {
      StopWatch sw = StopWatch.start("full log reload");
      Collection<List<? extends TimedVcsCommit>> commits = readFullLogFromVcs();
      List<? extends TimedVcsCommit> compoundLog = compound(commits);
      List<GraphCommit<Integer>> graphCommits = compactCommits(compoundLog, myHashMap);
      Map<VirtualFile, Collection<VcsRef>> refMap = loadRefsFromVcs(myProviders);
      PermanentGraph<Integer> permanentGraph = DataPack.buildPermanentGraph(graphCommits, new RefsModel(refMap, myHashMap.asIndexGetter()),
                                                                            myHashMap.asIndexGetter(),
                                                                            myHashMap.asHashGetter(), myProviders);
      sw.report();
      return Pair.create(permanentGraph, refMap);
    }

    @NotNull
    private Collection<List<? extends TimedVcsCommit>> readFullLogFromVcs() throws VcsException {
      final StopWatch sw = StopWatch.start("read full log from VCS");
      final Collection<List<? extends TimedVcsCommit>> commits = ContainerUtil.newArrayList();
      new ProviderIterator() {
        @Override
        void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException {
          commits.add(provider.readAllHashes(root, new Consumer<VcsUser>() {
            @Override
            public void consume(@NotNull VcsUser user) {
              myUserRegistry.addUser(user);
            }
          }));
          sw.rootCompleted(root);
        }
      }.iterate(myProviders);
      sw.report();
      return commits;
    }
  }

  private static class RefreshRequest {
    private static RefreshRequest RELOAD_ALL = new RefreshRequest(Collections.<VirtualFile>emptyList());
    @NotNull private final Collection<VirtualFile> rootsToRefresh;

    RefreshRequest(@NotNull Collection<VirtualFile> rootsToRefresh) {
      this.rootsToRefresh = rootsToRefresh;
    }
  }

  private static abstract class ProviderIterator {
    abstract void each(@NotNull VirtualFile root, @NotNull VcsLogProvider provider) throws VcsException;

    final void iterate(@NotNull Map<VirtualFile, VcsLogProvider> providers) throws VcsException {
      for (Map.Entry<VirtualFile, VcsLogProvider> entry : providers.entrySet()) {
        each(entry.getKey(), entry.getValue());
      }
    }
  }
}