diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,35 @@ # Change log for cut-the-crap +## Version 2.1.0 - 2020.10.11 + ++ Automatic cleanup of downloaded files + This is less annoying and also useful for the hosted version. ++ Add static executable support + This works for all linux versions, and allows us to drop ubuntu package support. ++ Add default output argument (convenience for debugging) ++ Expose download functions + +### Docs + ++ All shell commands run through same function + This allowed me to improve logging, which better shows what goes on. ++ Add better help docs for input format + We tell give hints on how to convert to mkv format. ++ Add default use case docs in readme. ++ Fix up readme docs with new cli + +### Cleanup + ++ nix: Move upstream patch to separate file and let default.nix depend on that ++ Drop support for Ubuntu ++ Delete segmentsize options which was unused. + +### Bug fixes + ++ Fix development shell to use nix cabal dependencies ++ Better default options that work with most videos. ++ Set the merge format for youtube dl to always be mkv to prevent issue (fixing the file not found bug) + ## Version 2.0.0 - 2020.10.09 - Better install instructions diff --git a/Readme.md b/Readme.md --- a/Readme.md +++ b/Readme.md @@ -30,7 +30,7 @@ # Install -## From source +## From source Install the [nix package](https://nixos.org/download.html) manager. ```shell @@ -48,21 +48,44 @@ This only works for nixpkgs that have cut-the-crap >= 1.4.2 or =< 1.3 There were some build issues with 1.4.0 and 1.4.1 (now fixed) -## Ubuntu -Download the .deb file from the [release page](https://github.com/jappeace/cut-the-crap/releases). -Install with: +## Static build (other linux) +Download the executable from the [release page](https://github.com/jappeace/cut-the-crap/releases). -```shell -apt install ffmpeg -dpkg -i ./cut-the-crap_1.0.0_amd64.deb +install ffmpeg and youtube-dl, for example: + ``` +apt install ffmpeg youtube-dl +``` -Execute with: -```shell +Now you can run the executable. + +# Usage notes + +Up to date help is available in the program itself: +``` cut-the-crap ``` -# Usage notes +Run the program: + +``` +cut-the-crap listen https://www.youtube.com/watch?v=_PB6Hdi4R7M +``` + +It works both with youtube or twitch videos (VODS). +The program simply passes the URL to [youtube-dl](https://github.com/ytdl-org/youtube-dl). + +We can also run it on a local file of course: +``` +cut-the-crap listen somelocalfile.mkv +``` + +There is also a work in progress subtitle generation: +``` +cut-the-crap subtitles https://www.youtube.com/watch?v=_PB6Hdi4R7M +``` + + ## Noise gate Make sure to record with a noise gate on your microphone. This will cut out background buzzing and allow you to use a more aggressive @@ -76,7 +99,7 @@ Then I can use: ```shell - cut-the-crap --inputFile ./recordFromObs.mkv --outputFile ./someOut.mkv --voiceTrack 2 --musicTrack 3 + cut-the-crap listen ./recordFromObs.mkv ./someOut.mkv --voiceTrack 2 --musicTrack 3 ``` So we throw away track 1, we use track 2 for silence detection, and track 3 get's mixed in after cutting is complete. diff --git a/cut-the-crap.cabal b/cut-the-crap.cabal --- a/cut-the-crap.cabal +++ b/cut-the-crap.cabal @@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack -- --- hash: 0f25306419959045fb637431751e1f2ae5212ba5c47b5ccc91dca4436a8a7943 +-- hash: 1e53cf8a33ea6d51825f52c3ba03f78a95655a38cc32028a5c83dad9d6ed390a name: cut-the-crap -version: 2.0.0 +version: 2.1.0 synopsis: Cuts out uninteresting parts of videos by detecting silences. description: Cut the crap is an automatic video editing program for streamers. It can cut out uninteresting parts by detecting silences. This was inspired by [jumpcutter](https://github.com/carykh/jumpcutter), where this program can get better quality results by using an (optional) dedicated microphone track. This prevents cutting of [quieter consonants](https://youtu.be/DQ8orIurGxw?t=675) for example. Using ffmpeg more efficiently also produces faster results and is less error prone. category: video @@ -27,6 +27,7 @@ Cut.Analyze Cut.Crap Cut.CutVideo + Cut.Download Cut.Options Cut.Shell Cut.SpeechRecognition @@ -36,7 +37,7 @@ src default-extensions: EmptyCase FlexibleContexts FlexibleInstances InstanceSigs MultiParamTypeClasses LambdaCase MultiWayIf NamedFieldPuns TupleSections DeriveFoldable DeriveFunctor DeriveGeneric DeriveLift DeriveTraversable DerivingStrategies GeneralizedNewtypeDeriving StandaloneDeriving OverloadedStrings TypeApplications ghc-options: -Wall -Wcompat -Wincomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Widentities - cc-options: -DMODELDIR="/nix/store/zr5cjjdqvv1vwixb0k5gg455n99cfhl8-pocketsphinx-5prealpha/share/pocketsphinx/model" + cc-options: -DMODELDIR="/nix/store/g91f3fkm432n552p2wmkg5rywhw39k3g-pocketsphinx-5prealpha/share/pocketsphinx/model" include-dirs: includes c-sources: @@ -45,10 +46,8 @@ pkgconfig-depends: pocketsphinx , sphinxbase - build-tools: - c2hs build-depends: - base >=4.7 && <5 + base >=4.7 && <4.15 , exceptions , generic-lens , lens @@ -72,7 +71,7 @@ app default-extensions: EmptyCase FlexibleContexts FlexibleInstances InstanceSigs MultiParamTypeClasses LambdaCase MultiWayIf NamedFieldPuns TupleSections DeriveFoldable DeriveFunctor DeriveGeneric DeriveLift DeriveTraversable DerivingStrategies GeneralizedNewtypeDeriving StandaloneDeriving OverloadedStrings TypeApplications ghc-options: -Wall -Wcompat -Wincomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Widentities -threaded -rtsopts -with-rtsopts=-N - cc-options: -DMODELDIR="/nix/store/zr5cjjdqvv1vwixb0k5gg455n99cfhl8-pocketsphinx-5prealpha/share/pocketsphinx/model" + cc-options: -DMODELDIR="/nix/store/g91f3fkm432n552p2wmkg5rywhw39k3g-pocketsphinx-5prealpha/share/pocketsphinx/model" include-dirs: includes c-sources: @@ -81,10 +80,8 @@ pkgconfig-depends: pocketsphinx , sphinxbase - build-tools: - c2hs build-depends: - base >=4.7 && <5 + base >=4.7 && <4.15 , cut-the-crap , exceptions , generic-lens @@ -111,6 +108,7 @@ Cut.Analyze Cut.Crap Cut.CutVideo + Cut.Download Cut.Options Cut.Shell Cut.SpeechRecognition @@ -120,7 +118,7 @@ src default-extensions: EmptyCase FlexibleContexts FlexibleInstances InstanceSigs MultiParamTypeClasses LambdaCase MultiWayIf NamedFieldPuns TupleSections DeriveFoldable DeriveFunctor DeriveGeneric DeriveLift DeriveTraversable DerivingStrategies GeneralizedNewtypeDeriving StandaloneDeriving OverloadedStrings TypeApplications ghc-options: -Wall -Wcompat -Wincomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Widentities - cc-options: -DMODELDIR="/nix/store/zr5cjjdqvv1vwixb0k5gg455n99cfhl8-pocketsphinx-5prealpha/share/pocketsphinx/model" + cc-options: -DMODELDIR="/nix/store/g91f3fkm432n552p2wmkg5rywhw39k3g-pocketsphinx-5prealpha/share/pocketsphinx/model" include-dirs: includes c-sources: @@ -129,11 +127,9 @@ pkgconfig-depends: pocketsphinx , sphinxbase - build-tools: - c2hs build-depends: QuickCheck - , base >=4.7 && <5 + , base >=4.7 && <4.15 , exceptions , generic-lens , hspec diff --git a/src/Cut/Crap.hs b/src/Cut/Crap.hs --- a/src/Cut/Crap.hs +++ b/src/Cut/Crap.hs @@ -35,65 +35,23 @@ import Options.Applicative import Shelly hiding (FilePath) import System.IO.Temp -import Network.URI(URI) -import System.Random -import Data.Word - -runYoutubeDL :: FileIO a -> URI -> IO FilePath -runYoutubeDL opts x = do - - inputNumbers :: Word32 <- randomIO - let inputChars :: String - -- youtube-dl doesn't do .mkv (not supported) - inputChars = (show inputNumbers) <> ".mp4" - filePath :: String - filePath = fromMaybe inputChars $ opts ^. work_dir - void $ shelly $ youtube_dl x filePath - pure filePath - - - -downloadIfNeccisary :: FileIO InputSource -> IO (FileIO FilePath) -downloadIfNeccisary x = do - result <- sequence $ (runYoutube <|> alreadyLocal) - case result of - Nothing -> error $ "Couldn't find " <> show x - Just y -> pure $ x & in_file .~ y - where - runYoutube :: Maybe (IO FilePath) - runYoutube = x ^? in_file . input_src_remote . to (runYoutubeDL x) - - alreadyLocal :: Maybe (IO FilePath) - alreadyLocal = preview (in_file . input_src_local_file . to pure) x - -downloadCutifNeccisary :: ListenCutOptionsT InputSource -> IO (ListenCutOptionsT FilePath) -downloadCutifNeccisary cut = repackRes - where - downloadRes :: IO (FileIO FilePath) - downloadRes = downloadIfNeccisary $ cut ^. lc_fileio - repackRes :: IO (ListenCutOptionsT (FilePath)) - repackRes = downloadRes <&> \y -> lc_fileio .~ y $ cut +import Cut.Download -- | reads settings from terminal and runs whatever command was -- given in program options entryPoint :: MonadMask m => MonadUnliftIO m => m () entryPoint = do result <- liftIO readSettings - betterResult <- liftIO $ case result of - ListenCut cut -> ListenCut <$> downloadCutifNeccisary cut - GenerateSubtitles x -> GenerateSubtitles <$> downloadIfNeccisary x - - -- I'm mr meeseeks look at me! - sequence_ $ betterResult ^? listen_cut_prism . to (void . runListenCut) - <|> betterResult ^? gnerate_sub_prism . to runGenSubs - pure () + case result of + ListenCut cut -> downloadCutifNeccisary cut $ void . runListenCut + GenerateSubtitles x -> downloadIfNeccisary x runGenSubs runGenSubs :: MonadIO m => FileIO FilePath -> m () runGenSubs options = liftIO $ withTempDir options $ \tmp -> do - betterOptions <- downloadCutifNeccisary simpleOptions - result <- shelly $ detectSpeech (set voice_track 1 betterOptions) tmp $ options ^. in_file - print result - traverse_ (T.writeFile (options ^. out_file) . makeSrt) result + downloadCutifNeccisary simpleOptions $ \betterOptions -> do + result <- shelly $ detectSpeech (set voice_track 1 betterOptions) tmp $ options ^. in_file + print result + traverse_ (T.writeFile (options ^. out_file) . makeSrt) result -- | Runs cut-the-crap with provided `ListenCutOptions` runListenCut :: MonadMask m => MonadUnliftIO m => ListenCutOptions -> m [Interval Sound] diff --git a/src/Cut/Download.hs b/src/Cut/Download.hs new file mode 100644 --- /dev/null +++ b/src/Cut/Download.hs @@ -0,0 +1,70 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeOperators #-} + +-- | Deal with downloading and the cli options involved +module Cut.Download + ( downloadIfNeccisary + , downloadCutifNeccisary + ) +where + +import Control.Lens +import Control.Monad +import Control.Monad.Catch +import Control.Monad.IO.Class +import Cut.Options +import Cut.Shell +import Data.Foldable (traverse_) +import Data.Word +import Network.URI (URI) +import Options.Applicative +import Shelly hiding (FilePath) +import System.Random + +-- | Downloads a URI to the filepath returned +runYoutubeDL :: FileIO a -> URI -> IO FilePath +runYoutubeDL opts x = do + -- we need to make our own because youtube-dl doesn't write to existing files + -- System.IO.Temp would've worked neatly here. + inputNumbers :: Word32 <- randomIO + let inputChars :: String + -- youtube-dl doesn't do .mkv (not supported) + inputChars = show inputNumbers + filePath :: String + filePath = maybe inputChars (flip () inputChars) $ opts ^. work_dir + void $ shelly $ youtube_dl x filePath + pure (filePath <> ".mkv") -- because 'youtube_dl' sets merge-output-format + +aquireFilePath :: FileIO InputSource -> IO (FileIO FilePath) +aquireFilePath x = do + result <- sequence $ (runYoutube <|> alreadyLocal) + case result of + Nothing -> error $ "Couldn't find " <> show x + Just y -> pure $ x & in_file .~ y + where + runYoutube :: Maybe (IO FilePath) + runYoutube = x ^? in_file . input_src_remote . to (runYoutubeDL x) + + alreadyLocal :: Maybe (IO FilePath) + alreadyLocal = preview (in_file . input_src_local_file . to pure) x + +removeDownloaded :: FileIO InputSource -> FileIO FilePath -> IO () +removeDownloaded cliInput aquiredPath = + traverse_ (const $ -- only do this if we got it from remote (we did the download) + traverse_ (shelly . rm) $ aquiredPath ^? in_file + ) $ cliInput ^? in_file . input_src_remote + +-- | 'FileIO' can have a remote, if so we download that and pass the downloaded +-- path to the continuation, if not we simply pass the input path to +-- the continuation. +-- We need a continuation to clean up the downloaded file. +downloadIfNeccisary :: MonadMask m => MonadIO m => FileIO InputSource -> (FileIO FilePath -> m a) -> m a +downloadIfNeccisary x = bracket (liftIO $ aquireFilePath x) (liftIO . removeDownloaded x) + +-- | this does the same as 'downloadIfNeccisary' but makes it work for +-- 'ListenCutOptionsT' with some type weaving +downloadCutifNeccisary :: MonadMask m => MonadIO m => ListenCutOptionsT InputSource -> ((ListenCutOptionsT FilePath) -> m a) -> m a +downloadCutifNeccisary cut fun = + downloadIfNeccisary (cut ^. lc_fileio) $ + fun . (\y -> lc_fileio .~ y $ cut) diff --git a/src/Cut/Options.hs b/src/Cut/Options.hs --- a/src/Cut/Options.hs +++ b/src/Cut/Options.hs @@ -22,7 +22,6 @@ -- * listen cut, options for video editing by audio , ListenCutOptionsT , ListenCutOptions - , seg_size , silent_treshold , detect_margin , voice_track @@ -66,10 +65,9 @@ simpleOptions :: ListenCutOptionsT InputSource simpleOptions = ListenCutOptions { lc_fileIO = simpleFileIO - , lc_segmentSize = _Just # def_seg_size , lc_silentTreshold = _Just # def_silent , lc_detectMargin = _Just # def_margin - , lc_voiceTrack = _Just # 2 + , lc_voiceTrack = _Just # def_voice_track , lc_musicTrack = Nothing , lc_silentDuration = _Just # def_duration , lc_cutNoise = def_cut_noise @@ -91,7 +89,6 @@ -- | Cut out by listening to sound options data ListenCutOptionsT a = ListenCutOptions { lc_fileIO :: FileIO a - , lc_segmentSize :: Maybe Int , lc_silentTreshold :: Maybe Double , lc_silentDuration :: Maybe Double , lc_detectMargin :: Maybe Double @@ -111,8 +108,8 @@ gnerate_sub_prism :: Prism' (ProgramOptions a) (FileIO a) gnerate_sub_prism = _Ctor @"GenerateSubtitles" -def_seg_size :: Int -def_seg_size = 20 +def_voice_track :: Int +def_voice_track = 1 def_margin :: Double def_margin = 0.05 @@ -121,7 +118,7 @@ def_cut_noise = False def_silent :: Double -def_silent = 0.0001 +def_silent = 0.075 def_duration :: Double def_duration = 0.25 @@ -132,9 +129,6 @@ lc_fileio :: Lens (ListenCutOptionsT a) (ListenCutOptionsT b) (FileIO a) (FileIO b) lc_fileio = field @"lc_fileIO" -seg_size :: Lens' (ListenCutOptionsT a) Int -seg_size = field @"lc_segmentSize" . non def_seg_size - detect_margin :: Lens' (ListenCutOptionsT a) Double detect_margin = field @"lc_detectMargin" . non def_margin @@ -182,8 +176,8 @@ parseFile :: Parser (FileIO InputSource) parseFile = FileIO - <$> argument readFileSource (metavar "INPUT" <> help "The input video, either a file or a uri") - <*> argument str (metavar "OUTPUT_FILE" <> help "The output name without format") + <$> argument readFileSource (metavar "INPUT" <> help "The input video, either a file or a uri. This program has tested best with the mkv container type, you can use ffmpeg to convert containers, for example \"ffmpeg -i input.mp4 output.mkv\", see https://opensource.com/article/17/6/ffmpeg-convert-media-file-formats") + <*> argument str (metavar "OUTPUT_FILE" <> help "The output name without format" <> value "out.mkv" <> showDefault) <*> optional (option str @@ -206,14 +200,10 @@ <*> optional (option auto - (long "segmentSize" <> help "The size of video segments in minutes") - ) - <*> optional - (option - auto ( long "silentTreshold" <> help - "The treshold for determining intersting sections, closer to zero is detects more audio (n: https://ffmpeg.org/ffmpeg-filters.html#silencedetect)" + "The treshold for determining intersting sections, closer to zero is detects more audio (n: https://ffmpeg.org/ffmpeg-filters.html#silencedetect), you may wish to tweak this variable a bit depending on your mic." + <> value def_silent <> showDefault ) ) <*> optional @@ -232,7 +222,7 @@ <*> optional (option auto - (long "voiceTrack" <> help "The track to detect the silences upon") + (long "voiceTrack" <> help "The track to detect the silences upon" <> value def_voice_track <> showDefault) ) <*> optional (option auto (long "musicTrack" <> help "The track to integrate")) diff --git a/src/Cut/Shell.hs b/src/Cut/Shell.hs --- a/src/Cut/Shell.hs +++ b/src/Cut/Shell.hs @@ -1,5 +1,7 @@ -- | Run shell programs +-- +-- We mostly add a lot of logging to figure out what goes on module Cut.Shell ( ffmpeg , ffmpeg' @@ -11,8 +13,10 @@ import Data.Text (Text) import qualified Data.Text as Text import Numeric -import Shelly +import Shelly hiding (run, command) import Network.URI(URI) +import Text.Printf +import Data.Time -- | Wrap ffmpeg for convenience and logging -- technically supports multiple inputs but for convenice we threw that. @@ -20,21 +24,27 @@ ffmpeg file args = ffmpeg' ("-y" : "-i" : Text.pack file : args) ffmpeg' :: [Text] -> Sh [Text] -ffmpeg' args = do - liftIO $ putStr "Running: " - liftIO $ print $ "ffmpeg " <> Text.unwords args - run_ "ffmpeg" args - Text.lines <$> lastStderr +ffmpeg' = run "ffmpeg" -- | Format floats for cli floatToText :: Double -> Text floatToText = Text.pack . flip (showFFloat (Just 10)) "" -youtube_dl :: URI -> FilePath -> Sh [Text] -youtube_dl uri path' = do - liftIO $ print $ "running youtube-dl " <> Text.unwords args - run_ "youtube-dl" args +run :: FilePath -> [Text] -> Sh [Text] +run command args = do + time <- liftIO getCurrentTime + let format = formatTime defaultTimeLocale "%F %T" time + liftIO $ putStrLn "--- " + liftIO $ printf "%s: %s %s" format command (Text.unwords args) + liftIO $ putStrLn " " -- flush + run_ command args + liftIO $ putStrLn "--- " Text.lines <$> lastStderr + +youtube_dl :: URI -> FilePath -> Sh [Text] +youtube_dl uri path' = run "youtube-dl" args where - args = [Text.pack $ show uri, - "-o", Text.pack path'] + args = [Text.pack $ show uri + , "-o", Text.pack path' + , "--merge-output-format", "mkv" + ]