State.hs

{-# LANGUAGE CPP #-}
-- Copyright (C) 2009 Petr Rockai
--           (C) 2012 José Neder
--
-- Permission is hereby granted, free of charge, to any person
-- obtaining a copy of this software and associated documentation
-- files (the "Software"), to deal in the Software without
-- restriction, including without limitation the rights to use, copy,
-- modify, merge, publish, distribute, sublicense, and/or sell copies
-- of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be
-- included in all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-- SOFTWARE.

module Darcs.Repository.State
    ( restrictSubpaths, restrictBoring, TreeFilter(..), restrictDarcsdir
    , maybeRestrictSubpaths
    -- * Diffs
    , unrecordedChanges, readPending
    -- * Trees
    , readRecorded, readUnrecorded, readRecordedAndPending, readWorking
    , readPendingAndWorking, readUnrecordedFiltered
    -- * Index
    , readIndex, updateIndex, invalidateIndex, UseIndex(..), ScanKnown(..)
    -- * Utilities
    , filterOutConflicts
    -- * Pending-related functions that depend on repo state
    , addPendingDiffToPending, addToPending
    ) where

import Prelude ()
import Darcs.Prelude

import Control.Monad ( when, foldM, forM )
import Control.Monad.State ( StateT, runStateT, get, put, liftIO )
import Control.Exception ( catch, IOException )
import Data.Maybe ( fromJust, isJust )
import Data.Ord ( comparing )
import Data.List ( sortBy, union, delete )
import Text.Regex( matchRegex )

import System.Directory( removeFile, doesFileExist, doesDirectoryExist, renameFile )
import System.FilePath
    ( (</>)
#if mingw32_HOST_OS
    , (<.>)
#endif
    )
import qualified Data.ByteString as B
    ( ByteString, readFile, drop, writeFile, empty, concat )
import qualified Data.ByteString.Char8 as BC
    ( pack, unpack, split )
import qualified Data.ByteString.Lazy as BL ( toChunks )

import Darcs.Patch ( RepoPatch, PrimOf, sortCoalesceFL, fromPrims
                   , PrimPatch, maybeApplyToTree
                   , tokreplace, forceTokReplace, move )
import Darcs.Patch.Named.Wrapped ( anonymous )
import Darcs.Patch.Apply ( ApplyState, applyToTree, effectOnFilePaths )
import Darcs.Patch.Witnesses.Ordered ( RL(..), FL(..), (+>+)
                                     , (:>)(..), reverseRL, reverseFL
                                     , mapFL, concatFL, toFL, nullFL )
import Darcs.Patch.Witnesses.Eq ( EqCheck(IsEq, NotEq) )
import Darcs.Patch.Witnesses.Unsafe ( unsafeCoerceP, unsafeCoercePEnd )
import Darcs.Patch.Witnesses.Sealed ( Sealed(Sealed), seal, unFreeLeft, mapSeal
                                    , freeGap, emptyGap, joinGap, FreeLeft, Gap(..) )
import Darcs.Patch.Commute ( selfCommuter, commuteFL )
import Darcs.Patch.CommuteFn ( commuterIdRL )
import Darcs.Patch.Permutations ( partitionConflictingFL, genCommuteWhatWeCanRL )
import Darcs.Patch.PatchInfoAnd ( PatchInfoAnd, n2pia )
import Darcs.Patch.FileHunk ( FileHunk(..), IsHunk(..) )
import Darcs.Patch.TokenReplace ( breakToTokens, defaultToks )

import Darcs.Repository.Flags ( UseIndex(..), ScanKnown(..), DiffAlgorithm(..)
                              , UpdateWorking(..), LookForMoves(..), LookForReplaces(..) )
import Darcs.Util.Global ( darcsdir )

import Darcs.Repository.InternalTypes ( Repository, repoFormat )
import Darcs.Repository.Format(formatHas, RepoProperty(NoWorkingDir))
import qualified Darcs.Repository.Pending as Pending
import Darcs.Repository.Prefs ( filetypeFunction, boringRegexps )
import Darcs.Repository.Diff ( treeDiff )

import Darcs.Util.Path
    ( AnchoredPath(..), anchorPath, floatPath, fn2fp
    , SubPath, sp2fn, filterPaths, FileName
    , parents, replacePrefixPath, anchoredRoot
    , toFilePath, simpleSubPath, normPath, floatSubPath, makeName
    )
import Darcs.Util.Hash( Hash( NoHash ) )
import Darcs.Util.Tree( Tree, restrict, FilterTree, expand, emptyTree, overlay, find
                      , ItemType(..), itemType, readBlob, modifyTree, findFile, TreeItem(..)
                      , makeBlobBS, expandPath )
import Darcs.Util.Tree.Plain( readPlainTree )
import Darcs.Util.Tree.Hashed( darcsTreeHash, readDarcsHashed, decodeDarcsHash, decodeDarcsSize )
import qualified Darcs.Util.Index as I
import qualified Darcs.Util.Tree as Tree
import Darcs.Util.Index ( listFileIDs, getFileID )

newtype TreeFilter m = TreeFilter { applyTreeFilter :: forall tr . FilterTree tr m => tr m -> tr m }

-- | From a repository and a list of SubPath's, construct a filter that can be
-- used on a Tree (recorded or unrecorded state) of this repository. This
-- constructed filter will take pending into account, so the subpaths will be
-- translated correctly relative to pending move patches.
restrictSubpaths :: (RepoPatch p, ApplyState p ~ Tree)
                 => Repository rt p wR wU wT -> [SubPath]
                 -> IO (TreeFilter m)
restrictSubpaths repo subpaths = do
  Sealed pending <- Pending.readPending repo
  restrictSubpathsAfter pending repo subpaths

-- | Like 'restrictSubpaths' but with the pending patch passed as a parameter.
-- The 'Repository' parameter is not used, we need it only to avoid
-- abiguous typing of @p@.
restrictSubpathsAfter :: (RepoPatch p, ApplyState p ~ Tree)
                      => FL (PrimOf p) wT wP
                      -> Repository rt p wR wU wT
                      -> [SubPath]
                      -> IO (TreeFilter m)
restrictSubpathsAfter pending _repo subpaths = do
  let paths = map (fn2fp . sp2fn) subpaths
      paths' = paths `union` effectOnFilePaths pending paths
      anchored = map floatPath paths'
      restrictPaths :: FilterTree tree m => tree m -> tree m
      restrictPaths = Tree.filter (filterPaths anchored)
  return (TreeFilter restrictPaths)

maybeRestrictSubpaths :: (RepoPatch p, ApplyState p ~ Tree)
                      => FL (PrimOf p) wT wP
                      -> Repository rt p wR wU wT
                      -> Maybe [SubPath]
                      -> IO (TreeFilter m)
maybeRestrictSubpaths pending repo =
  maybe (return $ TreeFilter id) (restrictSubpathsAfter pending repo)

-- |Is the given path in (or equal to) the _darcs metadata directory?
inDarcsDir :: AnchoredPath -> Bool
inDarcsDir (AnchoredPath (x:_)) | x == makeName darcsdir = True
inDarcsDir _ = False

-- | Construct a 'TreeFilter' that removes any boring files that are not also
-- contained in the argument 'Tree'.
--
-- The standard use case is for the argument to be the recorded state, possibly
-- with further patches applied, so as not to discard any files already known
-- to darcs. The result is usually applied to the full working state.
restrictBoring :: Tree m -> IO (TreeFilter m)
restrictBoring guide = do
  boring <- boringRegexps
  let boring' p | inDarcsDir p = False
      boring' p = not $ any (\rx -> isJust $ matchRegex rx p') boring
          where p' = anchorPath "" p
      restrictTree :: FilterTree t m => t m -> t m
      restrictTree = Tree.filter $ \p _ -> case find guide p of
                                             Nothing -> boring' p
                                             _ -> True
  return (TreeFilter restrictTree)

-- | Construct a Tree filter that removes any darcs metadata files the
-- Tree might have contained.
restrictDarcsdir :: TreeFilter m
restrictDarcsdir = TreeFilter $ Tree.filter $ \p _ -> not (inDarcsDir p)

{- |
For a repository and an optional list of paths (when 'Nothing', take
everything) compute a (forward) list of prims (i.e. a patch) going from the
recorded state of the repository (pristine) to the unrecorded state of the
repository (the working copy + pending). When a list of paths is given, at
least the files that live under any of these paths in either recorded or
unrecorded will be included in the resulting patch. NB. More patches may be
included in this list, eg. the full contents of the pending patch. This is
usually not a problem, since selectChanges will properly filter the results
anyway.

This also depends on the options given:

--look-for-moves: Detect pending file moves using the index. The resulting
  patches are added to pending and taken into consideration, when filtering
  the tree according to the given path list.

--look-for-adds: Include files in the working state that do not exist in the
  recorded + pending state.

--include-boring: Include even boring files.

--look-for-replaces: Detect pending replace patches. Like detected moves,
  these are added to the pending patch. Note that, like detected moves,
  these are mere proposals for the user to consider or reject.

--ignore-times: Disables index usage completely -- for each file, we read
  both the unrecorded and the recorded copy and run a diff on them. This is
  very inefficient, although in extremely rare cases, the index could go out
  of sync (file is modified, index is updated and file is modified again
  within a single second).

  Note that use of the index is also disabled when we detect moves or
  replaces, since this implies that the index is out of date.
-}
unrecordedChanges :: (RepoPatch p, ApplyState p ~ Tree)
                  => (UseIndex, ScanKnown, DiffAlgorithm)
                  -> LookForMoves
                  -> LookForReplaces
                  -> Repository rt p wR wU wT
                  -> Maybe [SubPath] -> IO (FL (PrimOf p) wT wU)
unrecordedChanges dopts lfm lfr r paths = do
  (pending :> working) <- readPendingAndWorking dopts lfm lfr r paths
  return $ sortCoalesceFL (pending +>+ working)

-- Implementation note: it is important to do things in the right order: we
-- first have to read the pending patch, then detect moves, then detect adds,
-- then detect replaces.
readPendingAndWorking :: forall rt p wR wU wT. (RepoPatch p, ApplyState p ~ Tree)
                      => (UseIndex, ScanKnown, DiffAlgorithm)
                      -> LookForMoves
                      -> LookForReplaces
                      -> Repository rt p wR wU wT
                      -> Maybe [SubPath]
                      -> IO ((FL (PrimOf p) :> FL (PrimOf p)) wT wU)
readPendingAndWorking _ _ _ r _ | formatHas NoWorkingDir (repoFormat r) = do
  IsEq <- return $ workDirLessRepoWitness r
  return (NilFL :> NilFL)
readPendingAndWorking (useidx, scan, diffalg) lfm lfr repo mbpaths = do
  (pending_tree, working_tree, pending) <-
    readPendingAndMovesAndUnrecorded repo useidx scan lfm mbpaths
  (pending_tree_with_replaces, Sealed replaces) <-
    getReplaces lfr diffalg repo pending_tree working_tree
  ft <- filetypeFunction
  wrapped_diff <- treeDiff diffalg ft pending_tree_with_replaces working_tree
  case unFreeLeft wrapped_diff of
    Sealed diff -> do
      return (pending +>+ unsafeCoercePEnd replaces :> unsafeCoercePEnd diff)

readPendingAndMovesAndUnrecorded
  :: (RepoPatch p, ApplyState p ~ Tree)
  => Repository rt p wR wU wT
  -> UseIndex
  -> ScanKnown
  -> LookForMoves
  -> Maybe [SubPath]
  -> IO ( Tree IO             -- pristine with (pending + moves)
        , Tree IO             -- working
        , FL (PrimOf p) wT wU -- pending + moves
        )
readPendingAndMovesAndUnrecorded repo useidx scan lfm mbpaths = do
  (pending_tree, Sealed pending) <- readPending repo
  moves <- getMoves lfm repo mbpaths
  let pending' = pending +>+ moves
  relevant <- maybeRestrictSubpaths pending' repo mbpaths
  pending_tree' <-
    applyTreeFilter relevant <$> applyToTree moves pending_tree
  let useidx' = if nullFL moves then useidx else IgnoreIndex
  index <-
    applyToTree moves =<< I.updateIndex =<<
    applyTreeFilter relevant <$> readIndex repo
  working_tree <- filteredWorking useidx' scan relevant index pending_tree'
  return (pending_tree', working_tree, unsafeCoercePEnd pending')

-- | @filteredWorking useidx scan relevant index pending_tree@ reads the
-- working tree and filters it according to options and @relevant@ file paths.
-- The @pending_tree@ is understood to have @relevant@ already applied and is
-- used (only) if @useidx == 'IgnoreIndex'@ and @scan == 'ScanKnown'@ to act as
-- a guide for filtering the working tree.
-- Note that even if @useidx '==' 'IgnoreIndex'@, the index is still used
-- to avoid filtering boring files that darcs knows about (see 'restrictBoring').
filteredWorking :: UseIndex
                -> ScanKnown
                -> TreeFilter IO
                -> Tree IO
                -> Tree IO
                -> IO (Tree IO)
filteredWorking useidx scan relevant index pending_tree = do
  applyTreeFilter restrictDarcsdir <$> case scan of
    ScanKnown -> case useidx of
      UseIndex -> return index
      IgnoreIndex -> do
        guide <- expand pending_tree
        applyTreeFilter relevant . restrict guide <$> readPlainTree "."
    ScanAll -> do
      nonboring <- restrictBoring index
      plain <- applyTreeFilter relevant . applyTreeFilter nonboring <$> readPlainTree "."
      return $ case useidx of
        UseIndex -> plain `overlay` index
        IgnoreIndex -> plain
    ScanBoring -> do
      plain <- applyTreeFilter relevant <$> readPlainTree "."
      return $ case useidx of
        UseIndex -> plain `overlay` index
        IgnoreIndex -> plain

-- | Witnesses the fact that in the absence of a working directory, we
-- pretend that the working dir updates magically to the tentative state.
workDirLessRepoWitness :: Repository rt p wR wU wT -> EqCheck wU wT
workDirLessRepoWitness r
 | formatHas NoWorkingDir (repoFormat r) = unsafeCoerceP IsEq
 | otherwise                             = NotEq

-- | Obtains a Tree corresponding to the "recorded" state of the repository:
-- this is the same as the pristine cache, which is the same as the result of
-- applying all the repository's patches to an empty directory.
readRecorded :: Repository rt p wR wU wT -> IO (Tree IO)
readRecorded _repo = do
  let h_inventory = darcsdir </> "hashed_inventory"
  hashed <- doesFileExist h_inventory
  if hashed
     then do inv <- B.readFile h_inventory
             let linesInv = BC.split '\n' inv
             case linesInv of
               [] -> return emptyTree
               (pris_line:_) -> do
                          let hash = decodeDarcsHash $ B.drop 9 pris_line
                              size = decodeDarcsSize $ B.drop 9 pris_line
                          when (hash == NoHash) $
                              fail $ "Bad pristine root: " ++ show pris_line
                          readDarcsHashed (darcsdir </> "pristine.hashed") (size, hash)
     else do have_pristine <- doesDirectoryExist $ darcsdir </> "pristine"
             have_current <- doesDirectoryExist $ darcsdir </> "current"
             case (have_pristine, have_current) of
               (True, _) -> readPlainTree $ darcsdir </> "pristine"
               (False, True) -> readPlainTree $ darcsdir </> "current"
               (_, _) -> fail "No pristine tree is available!"

-- | Obtains a Tree corresponding to the "unrecorded" state of the repository:
-- the modified files of the working tree plus the "pending" patch.
-- The optional list of paths allows to restrict the query to a subtree.
--
-- Limiting the query may be more efficient, since hashes on the uninteresting
-- parts of the index do not need to go through an up-to-date check (which
-- involves a relatively expensive lstat(2) per file.
readUnrecorded :: (RepoPatch p, ApplyState p ~ Tree)
               => Repository rt p wR wU wT -> Maybe [SubPath] -> IO (Tree IO)
readUnrecorded repo mbpaths = do
  Sealed pending <- Pending.readPending repo
  relevant <- maybeRestrictSubpaths pending repo mbpaths
  readIndex repo >>= I.updateIndex . applyTreeFilter relevant

-- | A variant of 'readUnrecorded' that takes the UseIndex and ScanKnown
-- options into account, similar to 'readPendingAndWorking'. We are only
-- interested in the resulting tree, not the patch, so the 'DiffAlgorithm' option
-- is irrelevant.
readUnrecordedFiltered :: (RepoPatch p, ApplyState p ~ Tree)
                       => Repository rt p wR wU wT
                       -> UseIndex
                       -> ScanKnown
                       -> LookForMoves
                       -> Maybe [SubPath] -> IO (Tree IO)
readUnrecordedFiltered repo useidx scan lfm mbpaths = do
  (_, working_tree, _) <-
    readPendingAndMovesAndUnrecorded repo useidx scan lfm mbpaths
  return working_tree

-- | Obtains a Tree corresponding to the complete working copy of the
-- repository (modified and non-modified files).
readWorking :: IO (Tree IO)
readWorking = expand =<< (applyTreeFilter restrictDarcsdir <$> readPlainTree ".")

-- | Obtains the recorded 'Tree' with the pending patch applied.
readRecordedAndPending :: (RepoPatch p, ApplyState p ~ Tree)
                       => Repository rt p wR wU wT -> IO (Tree IO)
readRecordedAndPending repo = fst `fmap` readPending repo

-- | Obtains the recorded 'Tree' with the pending patch applied, plus
--   the pending patch itself. The pending patch should start at the
--   recorded state (we even verify that it applies, and degrade to
--   renaming pending and starting afresh if it doesn't), but we've set to
--   say it starts at the tentative state.
--
--   Question (Eric Kow) Is this a bug? Darcs.Repository.Pending.readPending
--   says it is
readPending :: (RepoPatch p, ApplyState p ~ Tree)
            => Repository rt p wR wU wT
            -> IO (Tree IO, Sealed (FL (PrimOf p) wT))
readPending repo = do
  pristine <- readRecorded repo
  Sealed pending <- Pending.readPending repo
  catch ((\t -> (t, seal pending)) <$> applyToTree pending pristine) $
    \(err :: IOException) -> do
       putStrLn $ "Yikes, pending has conflicts! " ++ show err
       putStrLn "Stashing the buggy pending as _darcs/patches/pending_buggy"
       renameFile (darcsdir </> "patches" </> "pending")
                  (darcsdir </> "patches" </> "pending_buggy")
       return (pristine, seal NilFL)

index_file, index_invalid :: FilePath
index_file = darcsdir </> "index"
index_invalid = darcsdir </> "index_invalid"

-- | Mark the existing index as invalid. This has to be called whenever the
-- listing of pristine changes and will cause darcs to update the index next
-- time it tries to read it. (NB. This is about files added and removed from
-- pristine: changes to file content in either pristine or working are handled
-- transparently by the index reading code.)
invalidateIndex :: t -> IO ()
invalidateIndex _ = B.writeFile index_invalid B.empty

readIndex :: (RepoPatch p, ApplyState p ~ Tree)
          => Repository rt p wR wU wT -> IO I.Index
readIndex repo = do
  (invalid, exists, formatValid) <- checkIndex
  if not exists || invalid || not formatValid
     then do pris <- readRecordedAndPending repo
             idx <- I.updateIndexFrom index_file darcsTreeHash pris
             when invalid $ removeFile index_invalid
             return idx
     else I.readIndex index_file darcsTreeHash

updateIndex :: (RepoPatch p, ApplyState p ~ Tree)
            => Repository rt p wR wU wT -> IO ()
updateIndex repo = do
  (invalid, _, _) <- checkIndex
  pris <- readRecordedAndPending repo
  _ <- I.updateIndexFrom index_file darcsTreeHash pris
  when invalid $ removeFile index_invalid

checkIndex :: IO (Bool, Bool, Bool)
checkIndex = do
  invalid <- doesFileExist $ index_invalid
  exists <- doesFileExist index_file
  formatValid <- if exists
                     then I.indexFormatValid index_file
                     else return True
  when (exists && not formatValid) $ do
-- TODO this conditional logic (rename or delete) is mirrored in
-- Darcs.Util.Index.updateIndexFrom and should be refactored
#if mingw32_HOST_OS
    renameFile index_file (index_file <.> "old")
#else
    removeFile index_file
#endif
  return (invalid, exists, formatValid)

-- |Remove any patches (+dependencies) from a sequence that
-- conflict with the recorded or unrecorded changes in a repo
filterOutConflicts
  :: (RepoPatch p, ApplyState p ~ Tree)
  => RL (PatchInfoAnd rt p) wX wT -- ^Recorded patches from repository, starting from
                                  --  same context as the patches to filter
  -> Repository rt p wR wU wT     -- ^Repository itself, used for grabbing
                                  --  unrecorded changes
  -> FL (PatchInfoAnd rt p) wX wZ -- ^Patches to filter
  -> IO (Bool, Sealed (FL (PatchInfoAnd rt p) wX))
                                  -- ^True iff any patches were removed,
                                  --  possibly filtered patches
filterOutConflicts us repository them
     = do let commuter = commuterIdRL selfCommuter
          unrec <- fmap n2pia . anonymous . fromPrims
                     =<< unrecordedChanges (UseIndex, ScanKnown, MyersDiff)
                          NoLookForMoves NoLookForReplaces repository Nothing
          them' :> rest <- return $ partitionConflictingFL commuter them (us :<: unrec)
          return (check rest, Sealed them')
  where check :: FL p wA wB -> Bool
        check NilFL = False
        check _ = True

-- | Automatically detect file moves using the index.
-- TODO: This function lies about the witnesses.
getMoves :: forall rt p wR wU wT wB prim.
            (RepoPatch p, ApplyState p ~ Tree, prim ~ PrimOf p)
         => LookForMoves
         -> Repository rt p wR wU wT
         -> Maybe [SubPath]
         -> IO (FL prim wB wB)
getMoves NoLookForMoves _ _ = return NilFL
getMoves YesLookForMoves repository files =
    mkMovesFL <$> getMovedFiles repository files
  where
    mkMovesFL [] = NilFL
    mkMovesFL ((a,b,_):xs) = move (anchorPath "" a) (anchorPath "" b) :>: mkMovesFL xs

    getMovedFiles :: Repository rt p wR wU wT
                  -> Maybe [SubPath]
                  -> IO [(AnchoredPath, AnchoredPath, ItemType)]
    getMovedFiles repo fs = do
        old <- sortBy (comparing snd) <$> (listFileIDs =<< readIndex repo)
        nonboring <- restrictBoring emptyTree
        let addIDs = foldM (\xs (p, it)-> do mfid <- getFileID p
                                             return $ case mfid of
                                               Nothing -> xs
                                               Just fid -> ((p, it), fid):xs) []
        new <- sortBy (comparing snd) <$>
                 (addIDs . map (\(a,b) -> (a, itemType b)) . Tree.list  =<<
                   expand =<< applyTreeFilter nonboring <$> readPlainTree ".")
        let match (x:xs) (y:ys)
              | snd x > snd y = match (x:xs) ys
              | snd x < snd y = match xs (y:ys)
              | snd (fst x) /= snd (fst y) = match xs ys
              | otherwise = (fst (fst x), fst (fst y), snd (fst x)):match xs ys
            match _ _ = []
            movedfiles = match old new
            fmovedfiles =
              case fs of
                Nothing -> movedfiles
                Just subpath ->
                  filter (\(f1, f2, _) -> any (`elem` selfiles) [f1, f2]) movedfiles
                  where selfiles = map (floatPath . toFilePath) subpath
        return (resolve fmovedfiles)

    resolve :: [(AnchoredPath, AnchoredPath, ItemType)]
            -> [(AnchoredPath, AnchoredPath, ItemType)]
    resolve xs = fixPaths $ sortMoves $ deleteCycles xs
      where
        -- Input relation is left-and-right-unique. Makes cycle detection easier.
        deleteCycles [] = []
        deleteCycles whole@( x@(start,_,_):rest)
            = if hasCycle start whole start
                  then deleteCycles (deleteFrom start whole [])
                  else x:deleteCycles rest
           where hasCycle current ((a',b',_):rest') first
                     | a' == current = b' == first || hasCycle b' whole first
                     | otherwise     = hasCycle current rest' first 
                 hasCycle _ [] _     = False
                 deleteFrom a (y@(a',b',_):ys) seen
                   | a == a'   = deleteFrom b' (seen++ys) []
                   | otherwise = deleteFrom a ys (y:seen)
                 deleteFrom _ [] seen = seen

        sortMoves []                           = []
        sortMoves whole@(current@(_,dest,_):_) =
              smallest:sortMoves (delete smallest whole)
              where
               smallest = follow dest whole current
               follow prevDest (y@(s,d,_):ys) currentSmallest
                 -- destination is source of another move
                 | prevDest == s             = follow d whole y
                 -- parent of destination is also destination of a move
                 | d `elem` parents prevDest = follow d whole y
                 | otherwise     = follow prevDest ys currentSmallest
               follow _ [] currentSmallest = currentSmallest

        -- rewrite [d/ -> e/, .., d/f -> e/h] to [d/ -> e/, .., e/f -> e/h]
        fixPaths [] = []
        fixPaths (y@(f1,f2,t):ys)
                        | f1 == f2         = fixPaths ys
                        | TreeType <- t    = y:fixPaths (map replacepp ys)
                        | otherwise        = y:fixPaths ys
         where replacepp i@(if1,if2,it) | nfst == anchoredRoot = i
                                        | otherwise = (nfst, if2, it)
                where nfst = replacePrefixPath f1 f2 if1

-- | Search for possible replaces between the recordedAndPending state
-- and the unrecorded (or working) state. Return a Sealed FL list of
-- replace patches to be applied to the recordedAndPending state.
getReplaces :: forall rt p wR wU wT
             . (RepoPatch p, ApplyState p ~ Tree)
            => LookForReplaces
            -> DiffAlgorithm
            -> Repository rt p wR wU wT
            -> Tree IO -- ^ pending tree (including possibly detected moves)
            -> Tree IO -- ^ working tree
            -> IO (Tree IO, -- new pending tree
                   Sealed (FL (PrimOf p) wU))
getReplaces NoLookForReplaces _ _ pending _ = return (pending, Sealed NilFL)
getReplaces YesLookForReplaces diffalg _repo pending working = do
    ftf <- filetypeFunction
    Sealed changes <- unFreeLeft <$> treeDiff diffalg ftf pending working
    let allModifiedTokens = concat $ mapFL modifiedTokens changes
        replaces = rmInvalidReplaces allModifiedTokens
    (patches, new_pending) <-
      flip runStateT pending $
        forM replaces $ \(f,a,b) ->
          doReplace defaultToks
            (fromJust $ simpleSubPath $ fn2fp $ normPath f)
            (BC.unpack a) (BC.unpack b)
    return (new_pending, mapSeal concatFL $ toFL patches)
  where
    modifiedTokens :: PrimOf p wX wY -> [(FileName, B.ByteString, B.ByteString)]
    modifiedTokens p = case isHunk p of
      Just (FileHunk f _ old new) ->
        map (\(a,b) -> (f, a, b)) (concatMap checkModified $
          filter (\(a,b) -> length a == length b) -- only keep lines with same number of tokens
            $ zip (map breakToTokens old) (map breakToTokens new))
      Nothing -> []

    -- from a pair of token lists, create a pair of modified token lists
    checkModified = filter (\(a,b) -> a/=b) . uncurry zip

    rmInvalidReplaces [] = []
    rmInvalidReplaces ((f,old,new):rs)
      | any (\(f',a,b) -> f' == f && old == a && b /= new) rs =
          -- inconsistency detected
          rmInvalidReplaces $ filter (\(f'',a',_) -> f'' /= f || a' /= old) rs
    rmInvalidReplaces (r:rs) = r:rmInvalidReplaces (filter (/=r) rs)

    doReplace toks f old new = do
        pend <- get
        mpend' <- liftIO $ maybeApplyToTree replacePatch pend
        case mpend' of
          Nothing -> getForceReplace f toks old new
          Just pend' -> do
            put pend'
            return $ joinGap (:>:) (freeGap replacePatch) (emptyGap NilFL)
      where
        replacePatch = tokreplace (toFilePath f) toks old new

    getForceReplace :: (PrimPatch prim, ApplyState prim ~ Tree)
                    => SubPath -> String -> String -> String
                    -> StateT (Tree IO) IO (FreeLeft (FL prim))
    getForceReplace f toks old new = do
        let path = floatSubPath f
        -- the tree here is the "current" pending state
        tree <- get
        -- It would be nice if we could fuse the two traversals here, that is,
        -- expandPath and findFile. OTOH it is debatable whether adding a new
        -- effectful version of findFile to Darcs.Util.Tree is justified.
        expandedTree <- liftIO $ expandPath tree path
        content <- case findFile expandedTree path of
          Just blob -> liftIO $ readBlob blob
          Nothing -> bug $ "getForceReplace: not in tree: " ++ show path
        let newcontent = forceTokReplace toks (BC.pack new) (BC.pack old)
                            (B.concat $ BL.toChunks content)
            tree' = modifyTree expandedTree path . Just . File $ makeBlobBS newcontent
        ftf <- liftIO $ filetypeFunction
        normaliseNewTokPatch <- liftIO $ treeDiff diffalg ftf expandedTree tree'
        -- make sure we can apply them to the pending state
        patches <- return $ joinGap (+>+) normaliseNewTokPatch $ freeGap $
            tokreplace (toFilePath f) toks old new :>: NilFL
        mtree'' <- case unFreeLeft patches of
            Sealed ps -> liftIO $ maybeApplyToTree ps tree
        case mtree'' of
            Nothing -> bug "getForceReplace: unable to apply detected force replaces"
            Just tree'' -> do
                put tree''
                return patches


-- | Add an 'FL' of patches started from the pending state to the pending patch.
-- TODO: add witnesses for pending so we can make the types precise: currently
-- the passed patch can be applied in any context, not just after pending.
addPendingDiffToPending :: (RepoPatch p, ApplyState p ~ Tree)
                          => Repository rt p wR wU wT -> UpdateWorking
                          -> FreeLeft (FL (PrimOf p)) -> IO ()
addPendingDiffToPending _ NoUpdateWorking  _ = return ()
addPendingDiffToPending repo uw@YesUpdateWorking newP = do
    (toPend :> _) <-
        readPendingAndWorking (UseIndex, ScanKnown, MyersDiff)
          NoLookForMoves NoLookForReplaces repo Nothing
    invalidateIndex repo
    case unFreeLeft newP of
        (Sealed p) -> do recordedState <- readRecorded repo
                         Pending.makeNewPending repo uw (toPend +>+ p) recordedState

-- | Add an 'FL' of patches starting from the working state to the pending patch,
-- including as much extra context as is necessary (context meaning
-- dependencies), by commuting the patches to be added past as much of the
-- changes between pending and working as is possible, and including anything
-- that doesn't commute, and the patch itself in the new pending patch.
addToPending :: (RepoPatch p, ApplyState p ~ Tree)
             => Repository rt p wR wU wT -> UpdateWorking
             -> FL (PrimOf p) wU wY -> IO ()
addToPending _ NoUpdateWorking  _ = return ()
addToPending repo uw@YesUpdateWorking p = do
   (toPend :> toUnrec) <- readPendingAndWorking (UseIndex, ScanKnown, MyersDiff)
      NoLookForMoves NoLookForReplaces repo Nothing
   invalidateIndex repo
   case genCommuteWhatWeCanRL commuteFL (reverseFL toUnrec :> p) of
       (toP' :> p'  :> _excessUnrec) -> do
           recordedState <- readRecorded repo
           Pending.makeNewPending repo uw
            (toPend +>+ reverseRL toP' +>+ p') recordedState