diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,17 @@ +propellor (0.8.3) unstable; urgency=medium + + * The Debian package now includes a single-revision git repository in + /usr/src/propellor/, and ~/.propellor/ is set up to use this repository as + its origin remote. This avoids relying on the security of the github + repository when using the Debian package. + * The /usr/bin/propellor wrapper will warn when ~/.propellor/ is out of date + and a newer version is available, after which git merge upstream/master + can be run to merge it. + * Included the config.hs symlink to config-simple.hs in the cabal and Debian + packages. + + -- Joey Hess Fri, 22 Aug 2014 13:02:01 -0400 + propellor (0.8.2) unstable; urgency=medium * Fix bug in File.containsLines that caused lines that were already in the diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -22,9 +22,17 @@ install: install -d $(DESTDIR)/usr/bin $(DESTDIR)/usr/src/propellor install -s dist/build/propellor/propellor $(DESTDIR)/usr/bin/propellor + mkdir -p dist/gittmp $(CABAL) sdist - cat dist/propellor-*.tar.gz | \ - (cd $(DESTDIR)/usr/src/propellor && tar zx --strip-components=1) + cat dist/propellor-*.tar.gz | (cd dist/gittmp && tar zx --strip-components=1) + # cabal sdist does not preserve symlinks, so copy over file + cd dist/gittmp && for f in $$(find -type f); do rm -f $$f; cp -a ../../$$f $$f; done + cd dist/gittmp && git init && \ + git add . \ + && git commit -q -m "distributed version of propellor" \ + && git bundle create $(DESTDIR)/usr/src/propellor/propellor.git master HEAD \ + && git show-ref master --hash > $(DESTDIR)/usr/src/propellor/head + rm -rf dist/gittmp clean: rm -rf dist Setup tags propellor privdata/local diff --git a/config-joey.hs b/config-joey.hs --- a/config-joey.hs +++ b/config-joey.hs @@ -13,7 +13,6 @@ import qualified Propellor.Property.Sudo as Sudo import qualified Propellor.Property.User as User import qualified Propellor.Property.Hostname as Hostname ---import qualified Propellor.Property.Reboot as Reboot import qualified Propellor.Property.Tor as Tor import qualified Propellor.Property.Dns as Dns import qualified Propellor.Property.OpenId as OpenId @@ -21,7 +20,6 @@ import qualified Propellor.Property.Git as Git import qualified Propellor.Property.Apache as Apache import qualified Propellor.Property.Postfix as Postfix -import qualified Propellor.Property.Service as Service import qualified Propellor.Property.Grub as Grub import qualified Propellor.Property.Obnam as Obnam import qualified Propellor.Property.HostingProvider.DigitalOcean as DigitalOcean @@ -57,6 +55,12 @@ & Docker.configured & Docker.garbageCollected `period` Daily + + -- ssh on some extra ports to deal with horrible networks + -- while travelling + & alias "travelling.kitenet.net" + & Ssh.listenPort 80 + & Ssh.listenPort 443 -- Orca is the main git-annex build box. , standardSystem "orca.kitenet.net" Unstable "amd64" @@ -69,9 +73,8 @@ & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" & Docker.docked hosts "android-git-annex-builder" - -- not currently working - ! Docker.docked hosts "armel-git-annex-builder-companion" - ! Docker.docked hosts "armel-git-annex-builder" + & Docker.docked hosts "armel-git-annex-builder-companion" + & Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -161,6 +164,7 @@ "26fd6e38-1226-11e2-a75f-ff007033bdba" [] & JoeySites.twitRss + & JoeySites.pumpRss & alias "nntp.olduse.net" & alias "resources.olduse.net" @@ -233,13 +237,7 @@ -- Nothing is using http port 80, so listen on -- that port for ssh, for traveling on bad networks that -- block 22. - & "/etc/ssh/sshd_config" `File.containsLine` "Port 80" - `onChange` Service.restarted "ssh" - - -- temp - ! Docker.docked hosts "amd64-git-annex-builder" - ! Docker.docked hosts "i386-git-annex-builder" - ! Docker.docked hosts "android-git-annex-builder" + & Ssh.listenPort 80 --' __|II| ,. @@ -289,8 +287,6 @@ , let gitannexdir = GitAnnexBuilder.homedir "git-annex" in GitAnnexBuilder.androidContainer dockerImage "android-git-annex" doNothing gitannexdir & Docker.volume ("/home/joey/src/git-annex:" ++ gitannexdir) - - -- temp for an acquantance ] ++ monsters type Motd = [String] @@ -415,14 +411,13 @@ - /imap server - /pop server - /apache - - (need to re-rsync /srv/web to new kite.kitenet.net - - server before decommissioning) - bitlbee (EOL?) - prosody (EOL?) - ftpd (EOL) - - Pre-transition: - - re-rsync /home (skip ~joey and .pine*) + - cd /home && rsync -4 --progress -avz root@wren.kitenet.net:/home/ ./ --exclude='.pine*' --exclude='joey/*' --delete - - Transition plan: - - on darkstar: offlineimap run & disable cron job @@ -436,10 +431,11 @@ - - point wren.kitenet.net at kite.kitenet.net temporarily - (make real-wren.kitenet.net alias) - - reconfigure errol's email client to use new server - - - re-run offlinimap against new server + - - on darkstar: re-run offlinimap against new server - - test mail - - test virus filtering - - test http://kitenet.net/~kyle/ (user home dirs) + - - test mailman - - migrate user cron jobs -} , host "mouse.kitenet.net" diff --git a/config.hs b/config.hs new file mode 100644 --- /dev/null +++ b/config.hs @@ -0,0 +1,49 @@ +-- This is the main configuration file for Propellor, and is used to build +-- the propellor program. + +import Propellor +import Propellor.CmdLine +import Propellor.Property.Scheduled +import qualified Propellor.Property.File as File +import qualified Propellor.Property.Apt as Apt +import qualified Propellor.Property.Network as Network +--import qualified Propellor.Property.Ssh as Ssh +import qualified Propellor.Property.Cron as Cron +--import qualified Propellor.Property.Sudo as Sudo +import qualified Propellor.Property.User as User +--import qualified Propellor.Property.Hostname as Hostname +--import qualified Propellor.Property.Reboot as Reboot +--import qualified Propellor.Property.Tor as Tor +import qualified Propellor.Property.Docker as Docker + +main :: IO () +main = defaultMain hosts + +-- The hosts propellor knows about. +-- Edit this to configure propellor! +hosts :: [Host] +hosts = + [ host "mybox.example.com" + & os (System (Debian Unstable) "amd64") + & Apt.stdSourcesList + & Apt.unattendedUpgrades + & Apt.installed ["etckeeper"] + & Apt.installed ["ssh"] + & User.hasSomePassword "root" (Context "mybox.example.com") + & Network.ipv6to4 + & File.dirExists "/var/www" + & Docker.docked hosts "webserver" + & Docker.garbageCollected `period` Daily + & Cron.runPropellor "30 * * * *" + + -- A generic webserver in a Docker container. + , Docker.container "webserver" "joeyh/debian-stable" + & os (System (Debian Stable) "amd64") + & Apt.stdSourcesList + & Docker.publish "80:80" + & Docker.volume "/var/www:/var/www" + & Apt.serviceInstalledRunning "apache2" + + -- add more hosts here... + --, host "foo.example.com" = ... + ] diff --git a/debian/README.Debian b/debian/README.Debian --- a/debian/README.Debian +++ b/debian/README.Debian @@ -5,3 +5,10 @@ /usr/bin/propellor is a wrapper which will set up a propellor git repository in ~/.propellor/, and run ~/.propellor/propellor if it exists. Edit ~/.propellor/config.hs to configure it. + +Note that upgrading the propellor package will not update your +~/.propellor/ repository. This is because you may have local changes +to the source, or may need to adapt your config.hs to work with the new +version of propellor. Instead, if your ~/.propellor/ is from an older +version of propellor, /usr/bin/propellor will warn that it's out of date, +and tell you how to merge in the changes. diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +propellor (0.8.3) unstable; urgency=medium + + * The Debian package now includes a single-revision git repository in + /usr/src/propellor/, and ~/.propellor/ is set up to use this repository as + its origin remote. This avoids relying on the security of the github + repository when using the Debian package. + * The /usr/bin/propellor wrapper will warn when ~/.propellor/ is out of date + and a newer version is available, after which git merge upstream/master + can be run to merge it. + * Included the config.hs symlink to config-simple.hs in the cabal and Debian + packages. + + -- Joey Hess Fri, 22 Aug 2014 13:02:01 -0400 + propellor (0.8.2) unstable; urgency=medium * Fix bug in File.containsLines that caused lines that were already in the diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -3,6 +3,7 @@ Priority: optional Build-Depends: debhelper (>= 9), + git, ghc (>= 7.4), cabal-install, libghc-async-dev, diff --git a/debian/lintian-overrides b/debian/lintian-overrides deleted file mode 100644 --- a/debian/lintian-overrides +++ /dev/null @@ -1,2 +0,0 @@ -# These files are used in a git repository that propellor sets up. -propellor: package-contains-vcs-control-file usr/src/propellor/.gitignore diff --git a/debian/propellor.1 b/debian/propellor.1 deleted file mode 100644 --- a/debian/propellor.1 +++ /dev/null @@ -1,15 +0,0 @@ -.\" -*- nroff -*- -.TH propellor 1 "Commands" -.SH NAME -propellor \- property-based host configuration management in haskell -.SH SYNOPSIS -.B propellor [options] host -.SH DESCRIPTION -.I propellor -is a property-based host configuration management program written -and configured in haskell. -.PP -The first time you run propellor, it will set up a ~/.propellor/ -repository. Edit ~/.propellor/config.hs to configure it. -.SH AUTHOR -Joey Hess diff --git a/debian/rules b/debian/rules --- a/debian/rules +++ b/debian/rules @@ -11,7 +11,7 @@ override_dh_installdocs: dh_installdocs doc/README.mdwn override_dh_installman: - dh_installman debian/propellor.1 + dh_installman propellor.1 # Not intended for use by anyone except the author. announcedir: diff --git a/propellor.1 b/propellor.1 new file mode 100644 --- /dev/null +++ b/propellor.1 @@ -0,0 +1,15 @@ +.\" -*- nroff -*- +.TH propellor 1 "Commands" +.SH NAME +propellor \- property-based host configuration management in haskell +.SH SYNOPSIS +.B propellor [options] host +.SH DESCRIPTION +.I propellor +is a property-based host configuration management program written +and configured in haskell. +.PP +The first time you run propellor, it will set up a ~/.propellor/ +repository. Edit ~/.propellor/config.hs to configure it. +.SH AUTHOR +Joey Hess diff --git a/propellor.cabal b/propellor.cabal --- a/propellor.cabal +++ b/propellor.cabal @@ -1,5 +1,5 @@ Name: propellor -Version: 0.8.2 +Version: 0.8.3 Cabal-Version: >= 1.6 License: BSD3 Maintainer: Joey Hess @@ -17,14 +17,14 @@ Makefile config-simple.hs config-joey.hs + config.hs + propellor.1 debian/changelog debian/README.Debian - debian/propellor.1 debian/compat debian/control debian/copyright debian/rules - debian/lintian-overrides Synopsis: property-based host configuration management in haskell Description: Propellor enures that the system it's run in satisfies a list of diff --git a/src/Propellor/Property/SiteSpecific/JoeySites.hs b/src/Propellor/Property/SiteSpecific/JoeySites.hs --- a/src/Propellor/Property/SiteSpecific/JoeySites.hs +++ b/src/Propellor/Property/SiteSpecific/JoeySites.hs @@ -312,6 +312,11 @@ feed url desc = Cron.job desc crontime "joey" dir $ "./twitRss " ++ shellEscape url ++ " > " ++ shellEscape ("../" ++ desc ++ ".rss") +-- Work around for expired ssl cert. +pumpRss :: Property +pumpRss = Cron.job "pump rss" "15 * * * *" "joey" "/srv/web/tmp.kitenet.net/" + "wget https://pump2rss.com/feed/joeyh@identi.ca.atom -O pump.atom --no-check-certificate 2>/dev/null" + ircBouncer :: Property ircBouncer = propertyList "IRC bouncer" [ Apt.installed ["znc"] diff --git a/src/Propellor/Property/Ssh.hs b/src/Propellor/Property/Ssh.hs --- a/src/Propellor/Property/Ssh.hs +++ b/src/Propellor/Property/Ssh.hs @@ -9,7 +9,8 @@ hostKey, keyImported, knownHost, - authorizedKeys + authorizedKeys, + listenPort ) where import Propellor @@ -164,3 +165,18 @@ [ File.ownerGroup f user user , File.ownerGroup (takeDirectory f) user user ] + +-- | Makes the ssh server listen on a given port, in addition to any other +-- ports it is configured to listen on. +-- +-- Revert to prevent it listening on a particular port. +listenPort :: Int -> RevertableProperty +listenPort port = RevertableProperty enable disable + where + portline = "Port " ++ show port + enable = sshdConfig `File.containsLine` portline + `describe` ("ssh listening on " ++ portline) + `onChange` restartSshd + disable = sshdConfig `File.lacksLine` portline + `describe` ("ssh not listening on " ++ portline) + `onChange` restartSshd diff --git a/src/wrapper.hs b/src/wrapper.hs --- a/src/wrapper.hs +++ b/src/wrapper.hs @@ -9,32 +9,44 @@ -- uses it to build the real propellor program (if not already built), -- and runs it. -- --- The source is either copied from /usr/src/propellor, or is cloned from --- git over the network. +-- The source is cloned from /usr/src/propellor when available, +-- or is cloned from git over the network. module Main where +import Propellor.Message import Utility.UserInfo import Utility.Monad import Utility.Process import Utility.SafeCommand -import Utility.Directory +import Utility.Exception import Control.Monad import Control.Monad.IfElse +import Control.Applicative import System.Directory import System.FilePath import System.Environment (getArgs) import System.Exit import System.Posix.Directory +import System.IO -srcdir :: FilePath -srcdir = "/usr/src/propellor" +distdir :: FilePath +distdir = "/usr/src/propellor" +distrepo :: FilePath +distrepo = distdir "propellor.git" + +disthead :: FilePath +disthead = distdir "head" + +upstreambranch :: String +upstreambranch = "upstream/master" + -- Using the github mirror of the main propellor repo because -- it is accessible over https for better security. -srcrepo :: String -srcrepo = "https://github.com/joeyh/propellor.git" +netrepo :: String +netrepo = "https://github.com/joeyh/propellor.git" main :: IO () main = do @@ -46,42 +58,37 @@ wrapper :: [String] -> FilePath -> FilePath -> IO () wrapper args propellordir propellorbin = do - unlessM (doesDirectoryExist propellordir) $ - makeRepo + ifM (doesDirectoryExist propellordir) + ( checkRepo + , makeRepo + ) buildruncfg where - chain = do - (_, _, _, pid) <- createProcess (proc propellorbin args) - exitWith =<< waitForProcess pid makeRepo = do putStrLn $ "Setting up your propellor repo in " ++ propellordir putStrLn "" - ifM (doesDirectoryExist srcdir) - ( do - void $ boolSystem "cp" [Param "-a", File srcdir, File propellordir] + ifM (doesFileExist distrepo <||> doesDirectoryExist distrepo) + ( do + void $ boolSystem "git" [Param "clone", File distrepo, File propellordir] + fetchUpstreamBranch propellordir distrepo changeWorkingDirectory propellordir - void $ boolSystem "git" [Param "init"] - void $ boolSystem "git" [Param "add", Param "."] - setuprepo True - , do - void $ boolSystem "git" [Param "clone", Param srcrepo, File propellordir] void $ boolSystem "git" [Param "remote", Param "rm", Param "origin"] - setuprepo False + , void $ boolSystem "git" [Param "clone", Param netrepo, File propellordir] ) - setuprepo fromsrcdir = do + + checkRepo = whenM (doesFileExist disthead) $ do + headrev <- takeWhile (/= '\n') <$> readFile disthead changeWorkingDirectory propellordir - whenM (doesDirectoryExist "privdata") $ - mapM_ nukeFile =<< dirContents "privdata" - void $ boolSystem "git" [Param "commit", Param "--allow-empty", Param "--quiet", Param "-m", Param "setting up propellor git repository"] - void $ boolSystem "git" [Param "remote", Param "add", Param "upstream", Param srcrepo] - -- Connect synthetic git repo with upstream history so - -- merging with upstream will work going forward. - -- Note -s ours is used to avoid getting any divergent - -- changes from upstream. - when fromsrcdir $ do - void $ boolSystem "git" [Param "fetch", Param "upstream"] - version <- readProcess "dpkg-query" ["--showformat", "${Version}", "--show", "propellor"] - void $ boolSystem "git" [Param "merge", Param "-s", Param "ours", Param version] + headknown <- catchMaybeIO $ + withQuietOutput createProcessSuccess $ + proc "git" ["log", headrev] + if (headknown == Nothing) + then setupupstreammaster headrev propellordir + else do + merged <- not . null <$> + readProcess "git" ["log", headrev ++ "..HEAD", "--ancestry-path"] + unless merged $ + warnoutofdate propellordir True buildruncfg = do changeWorkingDirectory propellordir ifM (boolSystem "make" [Param "build"]) @@ -91,3 +98,69 @@ chain , error "Propellor build failed." ) + chain = do + (_, _, _, pid) <- createProcess (proc propellorbin args) + exitWith =<< waitForProcess pid + +-- Passed the user's propellordir repository, makes upstream/master +-- be a usefully mergeable branch. +-- +-- We cannot just use origin/master, because in the case of a distrepo, +-- it only contains 1 commit. So, trying to merge with it will result +-- in lots of merge conflicts, since git cannot find a common parent +-- commit. +-- +-- Instead, the upstream/master branch is created by taking the +-- upstream/master branch (which must be an old version of propellor, +-- as distributed), and diffing from it to the current origin/master, +-- and committing the result. This is done in a temporary clone of the +-- repository, giving it a new master branch. That new branch is fetched +-- into the user's repository, as if fetching from a upstream remote, +-- yielding a new upstream/master branch. +setupupstreammaster :: String -> FilePath -> IO () +setupupstreammaster newref propellordir = do + changeWorkingDirectory propellordir + go =<< catchMaybeIO getoldrev + where + go Nothing = warnoutofdate propellordir False + go (Just oldref) = do + let tmprepo = ".git/propellordisttmp" + let cleantmprepo = void $ catchMaybeIO $ removeDirectoryRecursive tmprepo + cleantmprepo + git ["clone", "--quiet", ".", tmprepo] + + changeWorkingDirectory tmprepo + git ["fetch", distrepo, "--quiet"] + git ["reset", "--hard", oldref, "--quiet"] + git ["merge", newref, "-s", "recursive", "-Xtheirs", "--quiet", "-m", "merging upstream version"] + + fetchUpstreamBranch propellordir tmprepo + cleantmprepo + warnoutofdate propellordir True + + getoldrev = takeWhile (/= '\n') + <$> readProcess "git" ["show-ref", upstreambranch, "--hash"] + + git = run "git" + run cmd ps = unlessM (boolSystem cmd (map Param ps)) $ + error $ "Failed to run " ++ cmd ++ " " ++ show ps + +warnoutofdate :: FilePath -> Bool -> IO () +warnoutofdate propellordir havebranch = do + warningMessage ("** Your " ++ propellordir ++ " is out of date..") + let also s = hPutStrLn stderr (" " ++ s) + also ("A newer upstream version is available in " ++ distrepo) + if havebranch + then also ("To merge it, run: git merge " ++ upstreambranch) + else also ("To merge it, find the most recent commit in your repository's history that corresponds to an upstream release of propellor, and set refs/remotes/" ++ upstreambranch ++ " to it. Then run propellor again.") + also "" + +fetchUpstreamBranch :: FilePath -> FilePath -> IO () +fetchUpstreamBranch propellordir repo = do + changeWorkingDirectory propellordir + void $ boolSystem "git" + [ Param "fetch" + , File repo + , Param ("+refs/heads/master:refs/remotes/" ++ upstreambranch) + , Param "--quiet" + ]