From 982bcfa2ad233e41e82b4d1319e280623b14a99c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Fri, 26 Dec 2014 13:43:44 +0200 Subject: [PATCH 01/90] WIP new upload --- ChangeLog.md | 4 + Stackage/PerformBuild.hs | 10 --- Stackage/Prelude.hs | 13 +++ Stackage/ServerBundle.hs | 173 +++++++++++++++++++++++++++++++++++++-- Stackage/Upload.hs | 129 +++++++++++++---------------- stackage.cabal | 3 +- 6 files changed, 243 insertions(+), 89 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 2a524e11..37704fa8 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +## 0.4.0 + +* Upload bundle V2 stuff + ## 0.3.1 * Added `justCheck` and `stackage check` command line. diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index 31602d55..5299f34e 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -412,16 +412,6 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = renameOrCopy :: FilePath -> FilePath -> IO () renameOrCopy src dest = rename src dest `catchIO` \_ -> copyDir src dest -copyDir :: FilePath -> FilePath -> IO () -copyDir src dest = - runResourceT $ sourceDirectoryDeep False src $$ mapM_C go - where - src' = src "" - go fp = forM_ (F.stripPrefix src' fp) $ \suffix -> do - let dest' = dest suffix - liftIO $ createTree $ parent dest' - sourceFile fp $$ (sinkFile dest' :: Sink ByteString (ResourceT IO) ()) - copyBuiltInHaddocks :: FilePath -> IO () copyBuiltInHaddocks docdir = do mghc <- findExecutable "ghc" diff --git a/Stackage/Prelude.hs b/Stackage/Prelude.hs index ab2187fc..28fff061 100644 --- a/Stackage/Prelude.hs +++ b/Stackage/Prelude.hs @@ -20,6 +20,9 @@ import Distribution.Version as X (Version (..), VersionRange) import Distribution.Version as X (withinRange) import qualified Distribution.Version as C +import Filesystem (createTree) +import Filesystem.Path (parent) +import qualified Filesystem.Path as F unPackageName :: PackageName -> Text unPackageName (PackageName str) = pack str @@ -101,3 +104,13 @@ topologicalSort toFinal toDeps = data TopologicalSortException key = NoEmptyDeps (Map key (Set key)) deriving (Show, Typeable) instance (Show key, Typeable key) => Exception (TopologicalSortException key) + +copyDir :: FilePath -> FilePath -> IO () +copyDir src dest = + runResourceT $ sourceDirectoryDeep False src $$ mapM_C go + where + src' = src "" + go fp = forM_ (F.stripPrefix src' fp) $ \suffix -> do + let dest' = dest suffix + liftIO $ createTree $ parent dest' + sourceFile fp $$ (sinkFile dest' :: Sink ByteString (ResourceT IO) ()) diff --git a/Stackage/ServerBundle.hs b/Stackage/ServerBundle.hs index 6cdf5cb4..5aedb090 100644 --- a/Stackage/ServerBundle.hs +++ b/Stackage/ServerBundle.hs @@ -7,21 +7,31 @@ module Stackage.ServerBundle , epochTime , bpAllPackages , docsListing + , createBundleV2 + , CreateBundleV2 (..) + , SnapshotType (..) + , writeIndexStyle + , DocMap + , PackageDocs (..) ) where import qualified Codec.Archive.Tar as Tar import qualified Codec.Archive.Tar.Entry as Tar import qualified Codec.Compression.GZip as GZip import qualified Data.Map as M +import Data.Aeson (ToJSON (..), (.=), object, FromJSON (..), (.:), withObject) +import System.IO.Temp (withTempDirectory) import qualified Data.Yaml as Y -import Filesystem (isFile) +import Filesystem (isFile, getWorkingDirectory, listDirectory, isDirectory, canonicalizePath) import Foreign.C.Types (CTime (CTime)) import Stackage.BuildConstraints import Stackage.BuildPlan import Stackage.Prelude +import System.IO.Temp (withTempDirectory) import qualified System.PosixCompat.Time as PC import qualified Text.XML as X import Text.XML.Cursor +import System.PosixCompat.Files (createSymbolicLink) -- | Get current time epochTime :: IO Tar.EpochTime @@ -73,13 +83,30 @@ serverBundle time title slug bp@BuildPlan {..} = GZip.compress $ Tar.write map (\(PackageName name) -> name) (M.keys $ siCorePackages bpSystemInfo) +-- | Package name is key +type DocMap = Map Text PackageDocs +data PackageDocs = PackageDocs + { pdVersion :: Text + , pdModules :: Map Text [Text] + -- ^ module name, path + } +instance ToJSON PackageDocs where + toJSON PackageDocs {..} = object + [ "version" .= pdVersion + , "modules" .= pdModules + ] +instance FromJSON PackageDocs where + parseJSON = withObject "PackageDocs" $ \o -> PackageDocs + <$> o .: "version" + <*> o .: "modules" + docsListing :: BuildPlan -> FilePath -- ^ docs directory - -> IO ByteString + -> IO DocMap docsListing bp docsDir = - fmap (Y.encode . fold) $ mapM go $ mapToList $ bpAllPackages bp + fmap fold $ mapM go $ mapToList $ bpAllPackages bp where - go :: (PackageName, Version) -> IO (Map Text Y.Value) + go :: (PackageName, Version) -> IO DocMap go (package, version) = do -- handleAny (const $ return mempty) $ do let dirname = fpFromText (concat [ display package @@ -107,8 +134,138 @@ docsListing bp docsDir = return $ if e then asMap $ singletonMap name [fpToText dirname, href] else mempty - return $ singletonMap (display package) $ Y.object - [ "version" Y..= display version - , "modules" Y..= m - ] + return $ singletonMap (display package) $ PackageDocs + { pdVersion = display version + , pdModules = m + } else return mempty + +data SnapshotType = STNightly + | STLTS !Int !Int -- ^ major, minor + deriving (Show, Read, Eq, Ord) + +instance ToJSON SnapshotType where + toJSON STNightly = object + [ "type" .= asText "nightly" + ] + toJSON (STLTS major minor) = object + [ "type" .= asText "lts" + , "major" .= major + , "minor" .= minor + ] +instance FromJSON SnapshotType where + parseJSON = withObject "SnapshotType" $ \o -> do + t <- o .: "type" + case asText t of + "nightly" -> return STNightly + "lts" -> STLTS + <$> o .: "major" + <*> o .: "minor" + _ -> fail $ "Unknown type for SnapshotType: " ++ unpack t + +data CreateBundleV2 = CreateBundleV2 + { cb2Plan :: BuildPlan + , cb2Type :: SnapshotType + , cb2DocsDir :: FilePath + , cb2Dest :: FilePath + } + +-- | Create a V2 bundle, which contains the build plan, metadata, docs, and doc +-- map. +createBundleV2 :: CreateBundleV2 -> IO () +createBundleV2 CreateBundleV2 {..} = do + docsDir <- canonicalizePath cb2DocsDir + docMap <- docsListing cb2Plan cb2DocsDir + + Y.encodeFile (fpToString $ docsDir "build-plan.yaml") cb2Plan + Y.encodeFile (fpToString $ docsDir "build-type.yaml") cb2Type + Y.encodeFile (fpToString $ docsDir "docs-map.yaml") docMap + void $ writeIndexStyle Nothing cb2DocsDir + + currentDir <- getWorkingDirectory + files <- listDirectory docsDir + + let args = "cfJ" + : fpToString (currentDir cb2Dest) + : "--dereference" + : map (fpToString . filename) files + cp = (proc "tar" args) { cwd = Just $ fpToString docsDir } + withCheckedProcess cp $ \ClosedStream Inherited Inherited -> return () + +writeIndexStyle :: Maybe Text -- ^ snapshot id + -> FilePath -- ^ docs dir + -> IO [String] +writeIndexStyle msnapid dir = do + dirs <- fmap sort + $ runResourceT + $ sourceDirectory dir + $$ filterMC (liftIO . isDirectory) + =$ mapC (fpToString . filename) + =$ sinkList + writeFile (dir "index.html") $ mkIndex + (unpack <$> msnapid) + dirs + writeFile (dir "style.css") styleCss + return dirs + +mkIndex :: Maybe String -> [String] -> String +mkIndex msnapid dirs = concat + [ "\nHaddocks index" + , "" + , "" + , "" + , "" + , "
" + , "
" + , "

Haddock documentation index

" + , flip foldMap msnapid $ \snapid -> concat + [ "

Return to snapshot

" + ] + , "
    " + , concatMap toLI dirs + , "
" + ] + where + toLI name = concat + [ "
  • " + , name + , "
    • " + ] + +styleCss :: String +styleCss = concat + [ "@media (min-width: 530px) {" + , "ul { -webkit-column-count: 2; -moz-column-count: 2; column-count: 2 }" + , "}" + , "@media (min-width: 760px) {" + , "ul { -webkit-column-count: 3; -moz-column-count: 3; column-count: 3 }" + , "}" + , "ul {" + , " margin-left: 0;" + , " padding-left: 0;" + , " list-style-type: none;" + , "}" + , "body {" + , " background: #f0f0f0;" + , " font-family: 'Lato', sans-serif;" + , " text-shadow: 1px 1px 1px #ffffff;" + , " font-size: 20px;" + , " line-height: 30px;" + , " padding-bottom: 5em;" + , "}" + , "h1 {" + , " font-weight: normal;" + , " color: #06537d;" + , " font-size: 45px;" + , "}" + , ".return a {" + , " color: #06537d;" + , " font-style: italic;" + , "}" + , ".return {" + , " margin-bottom: 1em;" + , "}"] diff --git a/Stackage/Upload.hs b/Stackage/Upload.hs index bb1c8246..bea1c289 100644 --- a/Stackage/Upload.hs +++ b/Stackage/Upload.hs @@ -13,17 +13,23 @@ module Stackage.Upload , uploadHackageDistro , UploadDocMap (..) , uploadDocMap + , uploadBundleV2 + , UploadBundleV2 (..) ) where import Control.Monad.Writer.Strict (execWriter, tell) import Data.Default.Class (Default (..)) +import Data.Function (fix) import Filesystem (isDirectory, isFile) import Network.HTTP.Client +import qualified Network.HTTP.Client.Conduit as HCC import Network.HTTP.Client.MultipartFormData import Stackage.BuildPlan (BuildPlan) import Stackage.Prelude -import Stackage.ServerBundle (bpAllPackages, docsListing) +import Stackage.ServerBundle (bpAllPackages, docsListing, writeIndexStyle) import System.IO.Temp (withSystemTempFile) +import qualified System.IO as IO +import qualified Data.Yaml as Y newtype StackageServer = StackageServer { unStackageServer :: Text } deriving (Show, Eq, Ord, Hashable, IsString) @@ -106,17 +112,7 @@ uploadDocs (UploadDocs (StackageServer host) fp0 token ident) man = do where uploadDocsDir = withSystemTempFile "haddocks.tar.xz" $ \fp h -> do hClose h - dirs <- fmap sort - $ runResourceT - $ sourceDirectory fp0 - $$ filterMC (liftIO . isDirectory) - =$ mapC (fpToString . filename) - =$ sinkList - writeFile (fp0 "index.html") $ mkIndex - (unpack $ unSnapshotIdent ident) - dirs - writeFile (fp0 "style.css") styleCss - -- FIXME write index.html, style.css + dirs <- writeIndexStyle (Just $ unSnapshotIdent ident) fp0 let cp = (proc "tar" $ "cJf" : fp : "index.html" : "style.css" : dirs) { cwd = Just $ fpToString fp0 } @@ -187,7 +183,7 @@ uploadDocMap :: UploadDocMap -> Manager -> IO (Response LByteString) uploadDocMap UploadDocMap {..} man = do docmap <- docsListing udmPlan udmDocDir req1 <- parseUrl $ unpack $ unStackageServer udmServer ++ "/upload-doc-map" - req2 <- formDataBody (formData docmap) req1 + req2 <- formDataBody (formData $ Y.encode docmap) req1 let req3 = req2 { method = "PUT" , requestHeaders = @@ -205,61 +201,54 @@ uploadDocMap UploadDocMap {..} man = do , partFileRequestBody "docmap" "docmap" $ RequestBodyBS docmap ] -mkIndex :: String -> [String] -> String -mkIndex snapid dirs = concat - [ "\nHaddocks index" - , "" - , "" - , "" - , "" - , "
    " - , "
    " - , "

    Haddock documentation index

    " - , "

    Return to snapshot

      " - , concatMap toLI dirs - , "
    " - ] - where - toLI name = concat - [ "
  • " - , name - , "
    • " - ] +data UploadBundleV2 = UploadBundleV2 + { ub2Server :: StackageServer + , ub2AuthToken :: Text + , ub2Bundle :: FilePath + } -styleCss :: String -styleCss = concat - [ "@media (min-width: 530px) {" - , "ul { -webkit-column-count: 2; -moz-column-count: 2; column-count: 2 }" - , "}" - , "@media (min-width: 760px) {" - , "ul { -webkit-column-count: 3; -moz-column-count: 3; column-count: 3 }" - , "}" - , "ul {" - , " margin-left: 0;" - , " padding-left: 0;" - , " list-style-type: none;" - , "}" - , "body {" - , " background: #f0f0f0;" - , " font-family: 'Lato', sans-serif;" - , " text-shadow: 1px 1px 1px #ffffff;" - , " font-size: 20px;" - , " line-height: 30px;" - , " padding-bottom: 5em;" - , "}" - , "h1 {" - , " font-weight: normal;" - , " color: #06537d;" - , " font-size: 45px;" - , "}" - , ".return a {" - , " color: #06537d;" - , " font-style: italic;" - , "}" - , ".return {" - , " margin-bottom: 1em;" - , "}"] +uploadBundleV2 :: UploadBundleV2 -> Manager -> IO Text +uploadBundleV2 UploadBundleV2 {..} man = IO.withBinaryFile (fpToString ub2Bundle) IO.ReadMode $ \h -> do + size <- IO.hFileSize h + req1 <- parseUrl $ unpack $ unStackageServer ub2Server ++ "/upload2" + let req2 = req1 + { method = "PUT" + , requestHeaders = + [ ("Authorization", encodeUtf8 ub2AuthToken) + , ("Accept", "application/json") + , ("Content-Type", "application/x-tar") + ] + , requestBody = HCC.requestBodySource (fromIntegral size) + $ sourceHandle h $= printProgress size + } + sink = decodeUtf8C =$ fix (\loop -> do + mx <- peekC + case mx of + Nothing -> error $ "uploadBundleV2: premature end of stream" + Just _ -> do + l <- lineC $ takeCE 4096 =$ foldC + let (cmd, msg') = break (== ':') l + msg = dropWhile (== ' ') $ dropWhile (== ':') msg' + case cmd of + "CONT" -> do + putStrLn msg + loop + "FAILURE" -> error $ "uploadBundleV2 failed: " ++ unpack msg + "SUCCESS" -> return msg + _ -> error $ "uploadBundleV2: unknown command " ++ unpack cmd + ) + withResponse req2 man $ \res -> HCC.bodyReaderSource (responseBody res) $$ sink + where + printProgress total = + loop 0 0 + where + loop sent lastPercent = + await >>= maybe (putStrLn "Upload complete") go + where + go bs = do + yield bs + let sent' = sent + fromIntegral (length bs) + percent = sent' * 100 `div` total + when (percent /= lastPercent) + $ putStrLn $ "Upload progress: " ++ tshow percent ++ "%" + loop sent' percent diff --git a/stackage.cabal b/stackage.cabal index 842cdbd7..d9e2a06d 100644 --- a/stackage.cabal +++ b/stackage.cabal @@ -1,5 +1,5 @@ name: stackage -version: 0.3.1 +version: 0.4.0 synopsis: "Stable Hackage," tools for creating a vetted set of packages from Hackage. description: Please see for a description and documentation. homepage: https://github.com/fpco/stackage @@ -52,6 +52,7 @@ library , yaml , unix-compat , http-client + , http-conduit , http-client-tls , temporary , data-default-class From 635fc73f55c71a94676198a0430bbf4d4267211a Mon Sep 17 00:00:00 2001 From: Leon Mergen Date: Tue, 17 Feb 2015 12:35:05 +0700 Subject: [PATCH 02/90] Update build-constraints.yaml --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index c4de7765..019851b6 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -703,6 +703,9 @@ packages: "Gabríel Arthúr Pétursson gabriel@system.is": - sdl2 + "Leon Mergen leon@solatis.com": + - network-attoparsec + "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 From 58a621bcbabe1d13133c9ac48e72d718c5fa6d8b Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Feb 2015 07:46:51 +0200 Subject: [PATCH 03/90] Remove no-longer-needed upper bound --- build-constraints.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 019851b6..e291aff3 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -750,7 +750,6 @@ packages: # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 - - resourcet < 1.1.4 # https://github.com/fpco/stackage/issues/445 - semigroupoids < 4.3 From 4cb98cd9ed782df8711af480ec53dce783a2452a Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Feb 2015 11:58:36 +0200 Subject: [PATCH 04/90] Include some missing system files --- debian-bootstrap.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/debian-bootstrap.sh b/debian-bootstrap.sh index c069dcb1..0beb5273 100755 --- a/debian-bootstrap.sh +++ b/debian-bootstrap.sh @@ -18,6 +18,9 @@ apt-get install -y \ build-essential \ libncurses-dev \ git \ + wget \ + m4 \ + texlive-binaries \ libgmp3c2 \ libgmp3-dev \ zlib1g-dev \ @@ -38,7 +41,6 @@ apt-get install -y \ libpq-dev \ libicu-dev \ libssl-dev \ - nettle-dev \ libgsl0-dev \ libblas-dev \ liblapack-dev \ @@ -50,3 +52,15 @@ apt-get install -y \ liblzma-dev \ libsdl2-dev \ libzmq3-dev + +mkdir /tmp/nettle-build +( +cd /tmp/nettle-build +wget https://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz +tar zxf nettle-2.7.1.tar.gz +cd nettle-2.7.1 +./configure +make +make install +) +rm -rf /tmp/nettle-build From 613fc07e8e6295520b49da940d6d7fbc9605e950 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Feb 2015 15:41:08 +0200 Subject: [PATCH 05/90] Skip test suite jberryman/directory-tree#4 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index e291aff3..a45cbfe3 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1011,6 +1011,9 @@ expected-test-failures: # https://github.com/haskell-distributed/distributed-process-execution/issues/2 - distributed-process-execution + # https://github.com/jberryman/directory-tree/issues/4 + - directory-tree + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 4761d55ce4c9646ab48bc232f1a34810583136c1 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Feb 2015 15:41:26 +0200 Subject: [PATCH 06/90] Docker image fixes --- build-constraints.yaml | 4 ++++ debian-bootstrap.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index a45cbfe3..9c5c2591 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1011,6 +1011,10 @@ expected-test-failures: # https://github.com/haskell-distributed/distributed-process-execution/issues/2 - distributed-process-execution + # Seems to depend on mtl being installed in user package database, which + # isn't always the case (e.g., build server) + - mtl + # https://github.com/jberryman/directory-tree/issues/4 - directory-tree diff --git a/debian-bootstrap.sh b/debian-bootstrap.sh index 0beb5273..0f296009 100755 --- a/debian-bootstrap.sh +++ b/debian-bootstrap.sh @@ -37,6 +37,7 @@ apt-get install -y \ llvm \ libbz2-dev \ libjudy-dev \ + libsqlite3-dev \ libmysqlclient-dev \ libpq-dev \ libicu-dev \ @@ -51,6 +52,7 @@ apt-get install -y \ libyaml-dev \ liblzma-dev \ libsdl2-dev \ + libxss-dev \ libzmq3-dev mkdir /tmp/nettle-build @@ -59,7 +61,7 @@ cd /tmp/nettle-build wget https://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz tar zxf nettle-2.7.1.tar.gz cd nettle-2.7.1 -./configure +./configure --prefix=/usr make make install ) From 5b72273a9f0fda115514f96589a658c5a25ed74f Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Feb 2015 16:38:07 +0200 Subject: [PATCH 07/90] Remove upper bounds and close #446 --- build-constraints.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 9c5c2591..0ab5775f 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -709,8 +709,7 @@ packages: "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 - - transformers-compat == 0.3.3.3 - - mtl-compat < 0.2 # https://github.com/fpco/stackage/issues/446 + - transformers-compat == 0.4.0.3 # https://github.com/fpco/stackage/issues/291 - random < 1.0.1.3 From 6c984bde6781febb7535780dd84665125a62cc54 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Feb 2015 06:47:36 +0200 Subject: [PATCH 08/90] Typo --- build-constraints.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 0ab5775f..6db0dd3f 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1012,7 +1012,7 @@ expected-test-failures: # Seems to depend on mtl being installed in user package database, which # isn't always the case (e.g., build server) - - mtl + - happy # https://github.com/jberryman/directory-tree/issues/4 - directory-tree From d4234b84668eed85b18b43191c2faff79ad05e79 Mon Sep 17 00:00:00 2001 From: Timothy Jones Date: Thu, 19 Feb 2015 17:13:23 +1300 Subject: [PATCH 09/90] Add http-media and cabal-test-quickcheck --- build-constraints.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 6db0dd3f..8eb473f6 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -706,6 +706,10 @@ packages: "Leon Mergen leon@solatis.com": - network-attoparsec + "Timothy Jones git@zmthy.io @zmthy": + - cabal-test-quickcheck + - http-media + "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 From bd6c774c02dfa256480fc7c977bf29bd0c423ae4 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 19 Feb 2015 07:51:43 +0200 Subject: [PATCH 10/90] Expected test failure massysett/multiarg#4 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 6db0dd3f..fb328637 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1017,6 +1017,9 @@ expected-test-failures: # https://github.com/jberryman/directory-tree/issues/4 - directory-tree + # https://github.com/massysett/multiarg/issues/4 + - multiarg + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 7183cf0f77f66a2463da2fd795654708dc9567ec Mon Sep 17 00:00:00 2001 From: Sebastiaan Visser Date: Thu, 19 Feb 2015 11:41:21 +0100 Subject: [PATCH 11/90] Update build-constraints.yaml --- build-constraints.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index fb328637..80e75e4e 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -390,9 +390,12 @@ packages: - fay-jquery - fay-text - fay-uri - - fclabels - snaplet-fay + "Sebastiaan Visser ": + - clay + - fclabels + "Rodrigo Setti ": - messagepack - messagepack-rpc From fb5e7af5c04cb158cb8cf2e8a6a8ca0ea57dfca3 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 19 Feb 2015 16:02:03 +0200 Subject: [PATCH 12/90] Remove expected failure --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 80e75e4e..4f9fbbb7 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1020,9 +1020,6 @@ expected-test-failures: # https://github.com/jberryman/directory-tree/issues/4 - directory-tree - # https://github.com/massysett/multiarg/issues/4 - - multiarg - # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 4312df025fd5aa4be9972c3c276945c9cfd65515 Mon Sep 17 00:00:00 2001 From: Oleg Grenrus Date: Sat, 21 Feb 2015 12:39:02 +0200 Subject: [PATCH 13/90] Add --skip-haddock -flag --- Stackage/CompleteBuild.hs | 2 ++ Stackage/InstallBuild.hs | 2 ++ Stackage/PerformBuild.hs | 3 ++- app/stackage.hs | 10 ++++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index cf74eef2..341be2b4 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -33,6 +33,7 @@ import System.IO (BufferMode (LineBuffering), hSetBuffering) -- | Flags passed in from the command line. data BuildFlags = BuildFlags { bfEnableTests :: !Bool + , bfEnableHaddock :: !Bool , bfDoUpload :: !Bool , bfEnableLibProfile :: !Bool , bfVerbose :: !Bool @@ -203,6 +204,7 @@ getPerformBuild buildFlags Settings {..} = PerformBuild , pbJobs = 8 , pbGlobalInstall = False , pbEnableTests = bfEnableTests buildFlags + , pbEnableHaddock = bfEnableHaddock buildFlags , pbEnableLibProfiling = bfEnableLibProfile buildFlags , pbVerbose = bfVerbose buildFlags , pbAllowNewer = bfSkipCheck buildFlags diff --git a/Stackage/InstallBuild.hs b/Stackage/InstallBuild.hs index abe8a14e..a6a6c25f 100644 --- a/Stackage/InstallBuild.hs +++ b/Stackage/InstallBuild.hs @@ -28,6 +28,7 @@ data InstallFlags = InstallFlags , ifJobs :: !Int , ifGlobalInstall :: !Bool , ifEnableTests :: !Bool + , ifEnableHaddock :: !Bool , ifEnableLibProfiling :: !Bool , ifVerbose :: !Bool , ifSkipCheck :: !Bool @@ -48,6 +49,7 @@ getPerformBuild plan InstallFlags{..} = , pbJobs = ifJobs , pbGlobalInstall = ifGlobalInstall , pbEnableTests = ifEnableTests + , pbEnableHaddock = ifEnableHaddock , pbEnableLibProfiling = ifEnableLibProfiling , pbVerbose = ifVerbose , pbAllowNewer = ifSkipCheck diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index c74b449a..e38f8e62 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -62,6 +62,7 @@ data PerformBuild = PerformBuild , pbGlobalInstall :: Bool -- ^ Register packages in the global database , pbEnableTests :: Bool + , pbEnableHaddock :: Bool , pbEnableLibProfiling :: Bool , pbVerbose :: Bool , pbAllowNewer :: Bool @@ -354,7 +355,7 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = -- dependency's haddocks before this finishes atomically $ putTMVar (piResult sbPackageInfo) True - when (pcHaddocks /= Don'tBuild && not (null $ sdModules $ ppDesc $ piPlan sbPackageInfo)) $ do + when (pbEnableHaddock && pcHaddocks /= Don'tBuild && not (null $ sdModules $ ppDesc $ piPlan sbPackageInfo)) $ do log' $ "Haddocks " ++ namever hfs <- readTVarIO sbHaddockFiles let hfsOpts = flip map (mapToList hfs) $ \(pkgVer, hf) -> concat diff --git a/app/stackage.hs b/app/stackage.hs index 8fdd2833..c8bb4f52 100644 --- a/app/stackage.hs +++ b/app/stackage.hs @@ -77,6 +77,11 @@ main = (switch (long "skip-tests" <> help "Skip build and running the test suites")) <*> + fmap + not + (switch + (long "skip-haddock" <> + help "Skip generating haddock documentation")) <*> fmap not (switch @@ -136,6 +141,11 @@ main = (switch (long "skip-tests" <> help "Skip build and running the test suites")) <*> + fmap + not + (switch + (long "skip-haddock" <> + help "Skip generating haddock documentation")) <*> switch (long "enable-library-profiling" <> help "Enable profiling when building") <*> From 4e2260a869ecc2288b927bf11fee67aca6708165 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 22 Feb 2015 07:03:40 +0200 Subject: [PATCH 14/90] Expected test failure zmthy/http-media#11 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 3fc076f6..3318f930 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1024,6 +1024,9 @@ expected-test-failures: # https://github.com/jberryman/directory-tree/issues/4 - directory-tree + # https://github.com/zmthy/http-media/issues/11 + - http-media + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 79b2b13b0e266e7461e66f5ebb7fd8f2f8068ea1 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 07:09:16 +0200 Subject: [PATCH 15/90] Upper bound for #453 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 3318f930..c9638349 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -760,6 +760,9 @@ packages: # https://github.com/fpco/stackage/issues/445 - semigroupoids < 4.3 + # https://github.com/fpco/stackage/issues/453 + - descriptive < 0.9 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From c7230cd1dbad30c85d79c5a5819a6a0f9754c552 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 07:10:35 +0200 Subject: [PATCH 16/90] Remove upper bounds and close #445 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index c9638349..1a3ea338 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -757,9 +757,6 @@ packages: # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 - # https://github.com/fpco/stackage/issues/445 - - semigroupoids < 4.3 - # https://github.com/fpco/stackage/issues/453 - descriptive < 0.9 From 56a37736690957a35eb3fcc1fc137459ef2fc9e1 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 07:11:54 +0200 Subject: [PATCH 17/90] Remove upper bound and close #402 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 1a3ea338..39f6feee 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -726,9 +726,6 @@ packages: # for language-ecmascript as well. - language-ecmascript < 0.17 - # https://github.com/fpco/stackage/issues/402 - - vector-space < 0.9 - # https://github.com/fpco/stackage/issues/407 - HStringTemplate < 0.8 From 42f96904d72e234e52dbe4761a77c05f1217d1cb Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 08:16:19 +0200 Subject: [PATCH 18/90] Temporary upper bound for byorgey/haxr#7 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 39f6feee..abf1e978 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -757,6 +757,9 @@ packages: # https://github.com/fpco/stackage/issues/453 - descriptive < 0.9 + # https://github.com/byorgey/haxr/issues/7 + - haxr < 3000.10.4 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 8a0e3b20aceb61cec6657360c89eba4c54d771c4 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 09:40:21 +0200 Subject: [PATCH 19/90] Add expected test failure ekmett/semigroupoids#18 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index abf1e978..0207887e 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1027,6 +1027,9 @@ expected-test-failures: # https://github.com/zmthy/http-media/issues/11 - http-media + # https://github.com/ekmett/semigroupoids/issues/18 + - semigroupoids + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 7be69244306fbb86361b775c3046b219357d73a2 Mon Sep 17 00:00:00 2001 From: Chris Done Date: Mon, 23 Feb 2015 10:54:42 +0100 Subject: [PATCH 20/90] Fix descriptive upper bound on hindent (fixes #453) --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 0207887e..953e4ef2 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -754,9 +754,6 @@ packages: # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 - # https://github.com/fpco/stackage/issues/453 - - descriptive < 0.9 - # https://github.com/byorgey/haxr/issues/7 - haxr < 3000.10.4 From 240de346305b902b564954e4c3bd750b72a8706b Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 16:30:34 +0200 Subject: [PATCH 21/90] Remove upper bound for byorgey/haxr#7 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 953e4ef2..8708a3c3 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -754,9 +754,6 @@ packages: # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 - # https://github.com/byorgey/haxr/issues/7 - - haxr < 3000.10.4 - # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 031ab4e77a0aa25e4856f6ffac8f67e084770c3e Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 16:33:10 +0200 Subject: [PATCH 22/90] Extra #442 upper bounds --- build-constraints.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 8708a3c3..b91604c2 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -750,6 +750,8 @@ packages: # https://github.com/fpco/stackage/issues/442 - blaze-builder < 0.4 + - blaze-markup < 0.7 + - blaze-html < 0.8 # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 From b7d0a219db3ec3f813cc8e2ad26e0264d525db2d Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 16:34:47 +0200 Subject: [PATCH 23/90] Extra upper bound for #443 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index b91604c2..123b4e8b 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -755,6 +755,9 @@ packages: # https://github.com/fpco/stackage/issues/443 - exceptions < 0.7 + - rest-client < 0.5 + - rest-types < 1.13 + - rest-core < 0.35 # Package flags are applied to individual packages, and override the values of # global-flags From 569e804bd4065f5aa71e485a659d013b1450516c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 23 Feb 2015 19:40:43 +0200 Subject: [PATCH 24/90] Extra upper bound to work around silkapp/rest#107 --- build-constraints.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 123b4e8b..5ebf663d 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -758,6 +758,7 @@ packages: - rest-client < 0.5 - rest-types < 1.13 - rest-core < 0.35 + - rest-gen < 0.17 # Package flags are applied to individual packages, and override the values of # global-flags From 350c4402742f51c70270c5ede6ce1987e003933f Mon Sep 17 00:00:00 2001 From: Greg V Date: Mon, 23 Feb 2015 21:40:54 +0300 Subject: [PATCH 25/90] Add gitson, pcre-heavy --- build-constraints.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 5ebf663d..8eeb83e1 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -712,6 +712,10 @@ packages: "Timothy Jones git@zmthy.io @zmthy": - cabal-test-quickcheck - http-media + + "Greg V greg@unrelenting.technology @myfreeweb": + - gitson + - pcre-heavy "Stackage upper bounds": From 4b4331e349ae071324bbb0e095c4b2f8d4c54c54 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 24 Feb 2015 07:00:45 +0200 Subject: [PATCH 26/90] Add @Gabriel439 as maintainer of pipes-safe See: https://github.com/fpco/stackage/issues/443#issuecomment-75662205 --- build-constraints.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 5ebf663d..ad7db534 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -379,6 +379,7 @@ packages: - pipes - pipes-parse - pipes-concurrency + - pipes-safe "Chris Allen ": - bloodhound From 4c1a2781173a1f8168a8a835974079888945225d Mon Sep 17 00:00:00 2001 From: Francesco Mazzoli Date: Tue, 24 Feb 2015 16:15:14 +0100 Subject: [PATCH 27/90] Add `language-c-quote` --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 6fd5222b..98bb6d7b 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -718,6 +718,9 @@ packages: - gitson - pcre-heavy + "Francesco Mazzoli f@mazzo.li @bitonic": + - language-c-quote + "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 From 4e1d67b34b787df988ec49d10ddd31589290bea7 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 25 Feb 2015 09:37:46 +0200 Subject: [PATCH 28/90] rest-wai upper bound --- build-constraints.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 98bb6d7b..8aeabfae 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -767,6 +767,7 @@ packages: - rest-types < 1.13 - rest-core < 0.35 - rest-gen < 0.17 + - rest-wai < 0.1.0.7 # Package flags are applied to individual packages, and override the values of # global-flags From 859415519567caf0857a2d2954f11e67b7713114 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 26 Feb 2015 07:16:24 +0200 Subject: [PATCH 29/90] Add back gitlib-libgit2 (fixes jwiegley/gitlib#23) --- build-constraints.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 8aeabfae..5feed0f1 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -578,6 +578,7 @@ packages: - haddock-api - here - hlibgit2 + - gitlib-libgit2 - hostname-validate - interpolatedstring-perl6 - iproute From 4af65f023229ef1e1584f57eec570d8c0214614c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Hahn?= Date: Fri, 27 Feb 2015 19:54:42 +0800 Subject: [PATCH 30/90] Add string-conversions --- build-constraints.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 8aeabfae..9b52943c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -713,7 +713,7 @@ packages: "Timothy Jones git@zmthy.io @zmthy": - cabal-test-quickcheck - http-media - + "Greg V greg@unrelenting.technology @myfreeweb": - gitson - pcre-heavy @@ -721,6 +721,9 @@ packages: "Francesco Mazzoli f@mazzo.li @bitonic": - language-c-quote + "Sönke Hahn soenkehahn@gmail.com @soenkehahn": + - string-conversions + "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 From fbf40bcd2e5e622d4babf0093fcc1e641c7d2eec Mon Sep 17 00:00:00 2001 From: marcinmrotek Date: Sun, 1 Mar 2015 08:29:37 +0100 Subject: [PATCH 31/90] Update build-constraints.yaml Added my libraries: diagrams-hsqml, type-list and vinyl-utils. --- build-constraints.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 28735ce9..5aefd126 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -603,6 +603,11 @@ packages: "Samplecount stefan@samplecount.com @kaoskorobase": - shake-language-c + + "Marcin Mrotek ": + - diagrams-hsqml + - type-list + - vinyl-utils "Stackage upper bounds": # https://github.com/fpco/stackage/issues/288 From b5ac384b46ffd3adf82d0a0c196c82c433adc363 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 1 Mar 2015 12:10:47 +0200 Subject: [PATCH 32/90] Create symlink for nettle --- debian-bootstrap.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian-bootstrap.sh b/debian-bootstrap.sh index 0f296009..7025ea05 100755 --- a/debian-bootstrap.sh +++ b/debian-bootstrap.sh @@ -64,5 +64,8 @@ cd nettle-2.7.1 ./configure --prefix=/usr make make install + +mkdir -p /usr/lib/x86_64-linux-gnu/ +ln -sfv /usr/lib/libnettle.so.4.7 /usr/lib/x86_64-linux-gnu/libnettle.so.4 ) rm -rf /tmp/nettle-build From dffc7edfba56a1bb7ce59c595f30e9a9687c4cdc Mon Sep 17 00:00:00 2001 From: Adam Bergmark Date: Sun, 1 Mar 2015 18:21:14 +0100 Subject: [PATCH 33/90] Add generic-xmlpickler and remove some duplicate entries --- build-constraints.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index fd5e83cd..e881afca 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -507,9 +507,9 @@ packages: - code-builder - fay-builder - generic-aeson + - generic-xmlpickler - hxt-pickle-utils - imagesize-conduit - - imagesize-conduit - json-schema - multipart - regular-xmlpickler @@ -522,8 +522,6 @@ packages: - rest-types - rest-wai - tostring - - tostring - - uri-encode - uri-encode "Simon Michael ": From a8a2bd77d34fa8c04c01b60d9dd345d1e41717b5 Mon Sep 17 00:00:00 2001 From: Dimitri 'phaazon' Sabadie Date: Tue, 3 Mar 2015 00:17:53 +0100 Subject: [PATCH 34/90] Added two packages for Dimitri Sabadie. - smoothie, a spline library ; - al, a raw binding to OpenAL 1.1. --- build-constraints.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index e881afca..bdb537b7 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -531,7 +531,9 @@ packages: - io-manager "Dimitri Sabadie ": - ghc-syb-utils @@ -643,7 +645,7 @@ packages: "Samplecount stefan@samplecount.com @kaoskorobase": - shake-language-c - + "Marcin Mrotek ": - diagrams-hsqml - type-list From 3807e55a52378132616e585b74a5572211041310 Mon Sep 17 00:00:00 2001 From: Boris Date: Tue, 3 Mar 2015 09:46:02 +0200 Subject: [PATCH 35/90] remove upper bound for https://github.com/d12frosted/CanonicalPath/issues/3 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index bdb537b7..e0d1391c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -758,9 +758,6 @@ packages: # https://github.com/fpco/stackage/issues/426 - utf8-string < 1 - # https://github.com/d12frosted/CanonicalPath/issues/3 - - system-canonicalpath < 0.3 - # https://github.com/fpco/stackage/issues/440 - th-orphans < 0.9 - file-location < 0.4.7 From d9905369565891335dc3184d06e8bb421f7162c2 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 3 Mar 2015 11:37:30 +0200 Subject: [PATCH 36/90] Temporarily disable al #461 --- build-constraints.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index e0d1391c..b19729a9 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -531,7 +531,8 @@ packages: - io-manager "Dimitri Sabadie Date: Tue, 3 Mar 2015 11:42:12 +0200 Subject: [PATCH 37/90] Expected test failure ndmitchell/hoogle#101 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index b19729a9..2fb473a9 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1046,6 +1046,9 @@ expected-test-failures: # https://github.com/ekmett/semigroupoids/issues/18 - semigroupoids + # https://github.com/ndmitchell/hoogle/issues/101 + - hoogle + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 2dcff03780800b5a6f8a58ea97255a1d3581fa29 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 3 Mar 2015 17:08:06 +0200 Subject: [PATCH 38/90] Expected test failure d12frosted/CanonicalPath#4 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 2fb473a9..897ca989 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1049,6 +1049,9 @@ expected-test-failures: # https://github.com/ndmitchell/hoogle/issues/101 - hoogle + # https://github.com/d12frosted/CanonicalPath/issues/4 + - system-canonicalpath + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 8dc931e093c65c8a6ba9ba87fb15c8f9d08ff390 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 3 Mar 2015 17:10:43 +0200 Subject: [PATCH 39/90] Expected test failure myfreeweb/gitson#1 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 897ca989..d021ca68 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1052,6 +1052,9 @@ expected-test-failures: # https://github.com/d12frosted/CanonicalPath/issues/4 - system-canonicalpath + # https://github.com/myfreeweb/gitson/issues/1 + - gitson + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From a22919f9402f9eea407fc676919221106bd1e819 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 5 Mar 2015 09:59:02 +0200 Subject: [PATCH 40/90] Temporarily block hopenpgp-tools #463 --- build-constraints.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index d021ca68..3e6c210f 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -477,7 +477,8 @@ packages: - openpgp-asciiarmor - MusicBrainz - DAV - - hopenpgp-tools + # https://github.com/fpco/stackage/issues/463 + #- hopenpgp-tools # https://github.com/fpco/stackage/issues/160 "Ketil Malde": From 0770ab8e27e62babba894c03ed5e4e5d33e55a93 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 5 Mar 2015 10:00:36 +0200 Subject: [PATCH 41/90] system-canonicalpath passes tests again --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 3e6c210f..d0801ef5 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1050,9 +1050,6 @@ expected-test-failures: # https://github.com/ndmitchell/hoogle/issues/101 - hoogle - # https://github.com/d12frosted/CanonicalPath/issues/4 - - system-canonicalpath - # https://github.com/myfreeweb/gitson/issues/1 - gitson From a3120f0d0925a1373828ba88a715c995a4dd2caf Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sat, 7 Mar 2015 19:09:59 +0200 Subject: [PATCH 42/90] Upper bound for #464 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index d0801ef5..3fbce4fd 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -777,6 +777,9 @@ packages: - rest-gen < 0.17 - rest-wai < 0.1.0.7 + # https://github.com/fpco/stackage/issues/464 + - profunctors < 4.4 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 6a492e8a68c64b8d6ae78c0e882396e48772efee Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sat, 7 Mar 2015 19:10:52 +0200 Subject: [PATCH 43/90] Upper bound for #465 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 3fbce4fd..c93239e9 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -780,6 +780,9 @@ packages: # https://github.com/fpco/stackage/issues/464 - profunctors < 4.4 + # https://github.com/fpco/stackage/issues/465 + - free < 4.11 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 1e004c73e4421328f5178f23fbd14d8af45ae4be Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sat, 7 Mar 2015 19:11:40 +0200 Subject: [PATCH 44/90] Upper bound for #466 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index c93239e9..01ec535e 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -783,6 +783,9 @@ packages: # https://github.com/fpco/stackage/issues/465 - free < 4.11 + # https://github.com/fpco/stackage/issues/466 + - linear < 1.17 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From a923f36160bf0ab0b4a925da1d79bcfeaa7d6833 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 8 Mar 2015 06:31:50 +0200 Subject: [PATCH 45/90] Upper bounds for #467 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 01ec535e..0221448c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -786,6 +786,9 @@ packages: # https://github.com/fpco/stackage/issues/466 - linear < 1.17 + # https://github.com/fpco/stackage/issues/467 + - lens < 4.8 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 0fd3665aea7914aaf3168978580e1f0f1a79ee81 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 8 Mar 2015 06:33:53 +0200 Subject: [PATCH 46/90] Expected test failure jcristovao/enclosed-exceptions#6 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 0221448c..34e66411 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1065,6 +1065,9 @@ expected-test-failures: # https://github.com/myfreeweb/gitson/issues/1 - gitson + # https://github.com/jcristovao/enclosed-exceptions/issues/6 + - enclosed-exceptions + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From 661ff0199932a015a7fa1d64c4f5b7b82ced9683 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 8 Mar 2015 06:42:48 +0200 Subject: [PATCH 47/90] Remove upper bounds and close #466 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 34e66411..e468214c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -783,9 +783,6 @@ packages: # https://github.com/fpco/stackage/issues/465 - free < 4.11 - # https://github.com/fpco/stackage/issues/466 - - linear < 1.17 - # https://github.com/fpco/stackage/issues/467 - lens < 4.8 From 7a10048b01e4e9ed712e3586a18eae37972cbec5 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 8 Mar 2015 06:43:12 +0200 Subject: [PATCH 48/90] Remove upper bounds and close #465 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index e468214c..2821a495 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -780,9 +780,6 @@ packages: # https://github.com/fpco/stackage/issues/464 - profunctors < 4.4 - # https://github.com/fpco/stackage/issues/465 - - free < 4.11 - # https://github.com/fpco/stackage/issues/467 - lens < 4.8 From 2495abb79e69126bbf13bf1e757a953d56c5a07a Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 8 Mar 2015 06:43:36 +0200 Subject: [PATCH 49/90] Remove upper bounds and close #464 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 2821a495..0684d999 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -777,9 +777,6 @@ packages: - rest-gen < 0.17 - rest-wai < 0.1.0.7 - # https://github.com/fpco/stackage/issues/464 - - profunctors < 4.4 - # https://github.com/fpco/stackage/issues/467 - lens < 4.8 From aab7687816fd5aed4284425e95fb0d508ba8f4fb Mon Sep 17 00:00:00 2001 From: Alexander Thiemann Date: Sun, 8 Mar 2015 20:33:54 +0100 Subject: [PATCH 50/90] add users, users-test, users-postgresql-simple --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 0684d999..8e143411 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -619,6 +619,9 @@ packages: - Spock - Spock-digestive - Spock-worker + - users + - users-test + - users-postgresql-simple "Joey Eremondi ": - aeson-pretty From ac07851dac4f836f70f234266b7eff5322d08dfc Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 10 Mar 2015 08:56:53 +0200 Subject: [PATCH 51/90] Expected test failure for users-postgresql-simple --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 8e143411..95e3a681 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1062,6 +1062,9 @@ expected-test-failures: # https://github.com/jcristovao/enclosed-exceptions/issues/6 - enclosed-exceptions + # Expects a running PostgreSQL server + - users-postgresql-simple + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From b096a8b49bd9d65c0107c94fc8b06a0b65fc4772 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 10 Mar 2015 08:59:25 +0200 Subject: [PATCH 52/90] Upper bound for #469 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 95e3a681..08a6de13 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -783,6 +783,9 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 + # https://github.com/fpco/stackage/issues/469 + - contravariant < 1.3 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 7f68c4aa0d057d3480e51f6f03b390043d2c4669 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 10 Mar 2015 21:39:30 +0200 Subject: [PATCH 53/90] Remove upper bounds and close #469 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 08a6de13..95e3a681 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -783,9 +783,6 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 - # https://github.com/fpco/stackage/issues/469 - - contravariant < 1.3 - # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 508fefe6186d346ac509c8189801a9acb8ab02bc Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 11 Mar 2015 07:27:48 +0200 Subject: [PATCH 54/90] Remove upper bounds and close #291 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 95e3a681..55c7d7b7 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -740,9 +740,6 @@ packages: # Force a specific version that's compatible with transformers 0.3 - transformers-compat == 0.4.0.3 - # https://github.com/fpco/stackage/issues/291 - - random < 1.0.1.3 - # https://github.com/fpco/stackage/issues/390 # NOTE: When this issue is resolved, remove the expected test failure # for language-ecmascript as well. From 4b9c9e1c68f14328d580c21f29dc208ac8e70484 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 11 Mar 2015 07:36:22 +0200 Subject: [PATCH 55/90] Get a missing executable for lhs2tex --- debian-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-bootstrap.sh b/debian-bootstrap.sh index 7025ea05..827e8460 100755 --- a/debian-bootstrap.sh +++ b/debian-bootstrap.sh @@ -20,7 +20,7 @@ apt-get install -y \ git \ wget \ m4 \ - texlive-binaries \ + texlive-full \ libgmp3c2 \ libgmp3-dev \ zlib1g-dev \ From 6443a042194fd3509088b103e8466360e36a9bb1 Mon Sep 17 00:00:00 2001 From: Oleg Grenrus Date: Wed, 11 Mar 2015 14:41:42 +0200 Subject: [PATCH 56/90] Add waitra package --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 55c7d7b7..d8e951d8 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -735,6 +735,9 @@ packages: "Sönke Hahn soenkehahn@gmail.com @soenkehahn": - string-conversions + "Oleg Grenrus oleg.grenrus@iki.fi @phadej": + - waitra + "Stackage upper bounds": # Force a specific version that's compatible with transformers 0.3 From d0f8ae6e34310e928292445b0a7e49925dc0bda4 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 11 Mar 2015 14:49:27 +0200 Subject: [PATCH 57/90] Expect nettle test failure --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 55c7d7b7..e3ee8c09 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1062,6 +1062,9 @@ expected-test-failures: # Expects a running PostgreSQL server - users-postgresql-simple + # Problems with linking with system libraries on Ubuntu 12.04 + - nettle + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From ba470b136b97f4255983a14c5f1fb7c4bb297c41 Mon Sep 17 00:00:00 2001 From: Tim Dysinger Date: Wed, 11 Mar 2015 09:50:57 -1000 Subject: [PATCH 58/90] Install alex & happy via PPA onto the docker image as a binary deb package Might be acceptable for #471 in the short term --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a83c42bc..c3956908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,9 @@ ADD debian-bootstrap.sh /tmp/debian-bootstrap.sh RUN DEBIAN_FRONTEND=noninteractive bash /tmp/debian-bootstrap.sh RUN rm /tmp/debian-bootstrap.sh -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install-1.20 ghc-7.8.4 +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install-1.20 ghc-7.8.4 alex-3.1.3 happy-1.19.4 -ENV PATH /home/stackage/.cabal/bin:/usr/local/sbin:/usr/local/bin:/opt/ghc/7.8.4/bin:/opt/cabal/1.20/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH /home/stackage/.cabal/bin:/usr/local/sbin:/usr/local/bin:/opt/ghc/7.8.4/bin:/opt/cabal/1.20/bin:/opt/alex/3.1.3/bin:/opt/happy/1.19.4/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN cabal update ADD . /tmp/stackage From 79ccb289bdde569f4ec148ae0f1b5781ab99be43 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 11 Mar 2015 22:11:32 +0200 Subject: [PATCH 59/90] Upper bound for #473 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index ef9b0fd4..3829e62f 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -783,6 +783,9 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 + # https://github.com/fpco/stackage/issues/473 + - snap < 0.14 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 3ef123f4b4ecf642c15f99264358c99aa92c35ae Mon Sep 17 00:00:00 2001 From: Leon Mergen Date: Thu, 12 Mar 2015 12:05:20 +0700 Subject: [PATCH 60/90] Adds network-anonymous-i2p --- build-constraints.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 3829e62f..bbd379d9 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -718,8 +718,9 @@ packages: "Gabríel Arthúr Pétursson gabriel@system.is": - sdl2 - "Leon Mergen leon@solatis.com": + "Leon Mergen leon@solatis.com @solatis": - network-attoparsec + - network-anonymous-i2p "Timothy Jones git@zmthy.io @zmthy": - cabal-test-quickcheck From a30a4297744681087bdeb55f24d15e93827d1d6a Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 08:12:35 +0200 Subject: [PATCH 61/90] Remove upper bounds and close #473 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index bbd379d9..7655b963 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -784,9 +784,6 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 - # https://github.com/fpco/stackage/issues/473 - - snap < 0.14 - # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 4b651e8865fe4ee9e551fc6b35582d33d01cfa42 Mon Sep 17 00:00:00 2001 From: Tim Dysinger Date: Wed, 11 Mar 2015 14:34:53 -1000 Subject: [PATCH 62/90] add performbuild flag for dynamic executables --- Stackage/CompleteBuild.hs | 1 + Stackage/InstallBuild.hs | 1 + Stackage/PerformBuild.hs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index 341be2b4..62a7634e 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -206,6 +206,7 @@ getPerformBuild buildFlags Settings {..} = PerformBuild , pbEnableTests = bfEnableTests buildFlags , pbEnableHaddock = bfEnableHaddock buildFlags , pbEnableLibProfiling = bfEnableLibProfile buildFlags + , pbEnableExecDyn = bfEnableExecDyn buildFlags , pbVerbose = bfVerbose buildFlags , pbAllowNewer = bfSkipCheck buildFlags } diff --git a/Stackage/InstallBuild.hs b/Stackage/InstallBuild.hs index a6a6c25f..74c78427 100644 --- a/Stackage/InstallBuild.hs +++ b/Stackage/InstallBuild.hs @@ -51,6 +51,7 @@ getPerformBuild plan InstallFlags{..} = , pbEnableTests = ifEnableTests , pbEnableHaddock = ifEnableHaddock , pbEnableLibProfiling = ifEnableLibProfiling + , pbEnableExecDyn = ifEnableExecDyn , pbVerbose = ifVerbose , pbAllowNewer = ifSkipCheck } diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index e38f8e62..e25094c6 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -64,6 +64,7 @@ data PerformBuild = PerformBuild , pbEnableTests :: Bool , pbEnableHaddock :: Bool , pbEnableLibProfiling :: Bool + , pbEnableExecDyn :: Bool , pbVerbose :: Bool , pbAllowNewer :: Bool -- ^ Pass --allow-newer to cabal configure @@ -317,6 +318,7 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = tell' $ "--flags=" ++ flags when (pbEnableLibProfiling && pcEnableLibProfile) $ tell' "--enable-library-profiling" + when pbEnableExecDyn $ tell' "--enable-executable-dynamic" where tell' x = tell (x:) From f6b95190f1f7420071e2dedc48ad9bc7cf32f31f Mon Sep 17 00:00:00 2001 From: Tim Dysinger Date: Wed, 11 Mar 2015 14:34:22 -1000 Subject: [PATCH 63/90] add installflag for dynamic executables --- Stackage/InstallBuild.hs | 1 + app/stackage.hs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Stackage/InstallBuild.hs b/Stackage/InstallBuild.hs index 74c78427..757939a7 100644 --- a/Stackage/InstallBuild.hs +++ b/Stackage/InstallBuild.hs @@ -30,6 +30,7 @@ data InstallFlags = InstallFlags , ifEnableTests :: !Bool , ifEnableHaddock :: !Bool , ifEnableLibProfiling :: !Bool + , ifEnableExecDyn :: !Bool , ifVerbose :: !Bool , ifSkipCheck :: !Bool } deriving (Show) diff --git a/app/stackage.hs b/app/stackage.hs index c8bb4f52..57497dfd 100644 --- a/app/stackage.hs +++ b/app/stackage.hs @@ -149,6 +149,9 @@ main = switch (long "enable-library-profiling" <> help "Enable profiling when building") <*> + switch + (long "enable-executable-dynamic" <> + help "Enable dynamic executables when building") <*> switch (long "verbose" <> short 'v' <> help "Output verbose detail about the build steps") <*> From 0ffdb203c3d340e2423b9d08b123491b94ef6225 Mon Sep 17 00:00:00 2001 From: Tim Dysinger Date: Wed, 11 Mar 2015 14:33:14 -1000 Subject: [PATCH 64/90] add buildflags for dynamic executables --- Stackage/CompleteBuild.hs | 1 + app/stackage.hs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index 62a7634e..ab9c3a3b 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -36,6 +36,7 @@ data BuildFlags = BuildFlags , bfEnableHaddock :: !Bool , bfDoUpload :: !Bool , bfEnableLibProfile :: !Bool + , bfEnableExecDyn :: !Bool , bfVerbose :: !Bool , bfSkipCheck :: !Bool } deriving (Show) diff --git a/app/stackage.hs b/app/stackage.hs index 57497dfd..1b9583c7 100644 --- a/app/stackage.hs +++ b/app/stackage.hs @@ -90,6 +90,9 @@ main = switch (long "enable-library-profiling" <> help "Enable profiling when building") <*> + switch + (long "enable-executable-dynamic" <> + help "Enable dynamic executables when building") <*> switch (long "verbose" <> short 'v' <> help "Output verbose detail about the build steps") <*> From eadca478ae0e3161308c64e0cc63e342ad3d303d Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 14:26:16 +0200 Subject: [PATCH 65/90] First stab an incremental builds Pinging @chrisdone --- Stackage/CompleteBuild.hs | 4 +- Stackage/GhcPkg.hs | 79 ++++++++++++++++++++++ Stackage/PerformBuild.hs | 137 +++++++++++++++++++++++++++++++------- stackage.cabal | 2 + 4 files changed, 195 insertions(+), 27 deletions(-) create mode 100644 Stackage/GhcPkg.hs diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index 341be2b4..97e8ba72 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -67,7 +67,7 @@ nightlySettings :: Text -- ^ day -> Settings nightlySettings day plan' = Settings { planFile = nightlyPlanFile day - , buildDir = fpFromText $ "builds/stackage-nightly-" ++ day + , buildDir = fpFromText $ "builds/nightly" , logDir = fpFromText $ "logs/stackage-nightly-" ++ day , title = \ghcVer -> concat [ "Stackage Nightly " @@ -120,7 +120,7 @@ getSettings man (LTS bumpType) = do return Settings { planFile = newfile - , buildDir = fpFromText $ "builds/stackage-lts-" ++ tshow new + , buildDir = fpFromText $ "builds/lts" , logDir = fpFromText $ "logs/stackage-lts-" ++ tshow new , title = \ghcVer -> concat [ "LTS Haskell " diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs new file mode 100644 index 00000000..4985d2f2 --- /dev/null +++ b/Stackage/GhcPkg.hs @@ -0,0 +1,79 @@ +{-# LANGUAGE NoImplicitPrelude #-} +-- | General commands related to ghc-pkg. + +module Stackage.GhcPkg where + +import Data.Conduit +import qualified Data.Conduit.List as CL +import Data.Conduit.Process +import qualified Data.Conduit.Text as CT +import Data.Maybe +import Data.Text (Text) +import qualified Data.Text as T +import Distribution.Compat.ReadP +import Distribution.Package +import Distribution.Text (parse) +import Filesystem.Path.CurrentOS (FilePath) +import qualified Filesystem.Path.CurrentOS as FP +import Data.Map (Map) +import Data.Version (Version) +import Stackage.Prelude + +setupPackageDatabase + :: Maybe FilePath -- ^ database location, Nothing if using global DB + -> Map PackageName Version -- ^ packages and versions to be installed + -> IO (Set PackageName) -- ^ packages remaining in the database after cleanup +setupPackageDatabase mdb toInstall = do + registered1 <- getRegisteredPackages flags + forM_ registered1 $ \(PackageIdentifier name version) -> + case lookup name toInstall of + Just version' | version /= version' -> unregisterPackage flags name + _ -> return () + broken <- getBrokenPackages flags + forM_ broken $ \(PackageIdentifier name _) -> unregisterPackage flags name + foldMap (\(PackageIdentifier name _) -> singletonSet name) + <$> getRegisteredPackages flags + where + flags = ghcPkgFlags mdb + +ghcPkgFlags :: Maybe FilePath -> [String] +ghcPkgFlags mdb = + "--no-user-package-db" : + case mdb of + Nothing -> ["--global"] + Just fp -> ["--package-db=" ++ fpToString fp] + +-- | Get broken packages. +getBrokenPackages :: [String] -> IO [PackageIdentifier] +getBrokenPackages flags = do + (_,ps) <- sourceProcessWithConsumer + (proc + "ghc-pkg" + ("check" : "--simple-output" : flags)) + (CT.decodeUtf8 $= CT.lines $= CL.consume) + return (mapMaybe parsePackageIdent (T.words (T.unlines ps))) + +-- | Get available packages. +getRegisteredPackages :: [String] -> IO [PackageIdentifier] +getRegisteredPackages flags = do + (_,ps) <- sourceProcessWithConsumer + (proc + "ghc-pkg" + ("list" : "--simple-output" : flags)) + (CT.decodeUtf8 $= CT.lines $= CL.consume) + return (mapMaybe parsePackageIdent (T.words (T.unlines ps))) + +-- | Parse a package identifier: foo-1.2.3 +parsePackageIdent :: Text -> Maybe PackageIdentifier +parsePackageIdent = fmap fst . + listToMaybe . + filter (null . snd) . + readP_to_S parse . T.unpack + +-- | Unregister a package. +unregisterPackage :: [String] -> PackageName -> IO () +unregisterPackage flags ident = do + void (readProcessWithExitCode + "ghc-pkg" + ("unregister": flags ++ ["--force", unpack $ display ident]) + "") diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index e38f8e62..fe8dac4a 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -19,11 +19,12 @@ import qualified Data.Map as Map import Data.NonNull (fromNullable) import Filesystem (canonicalizePath, createTree, getWorkingDirectory, isDirectory, - removeTree, rename) + removeTree, rename, isFile, removeFile) import Filesystem.Path (parent) import qualified Filesystem.Path as F import Stackage.BuildConstraints import Stackage.BuildPlan +import Stackage.GhcPkg import Stackage.PackageDescription import Stackage.Prelude hiding (pi) import System.Directory (findExecutable) @@ -134,6 +135,10 @@ pbLibDir pb = pbInstallDest pb "lib" pbDataDir pb = pbInstallDest pb "share" pbDocDir pb = pbInstallDest pb "doc" +-- | Directory keeping previous result info +pbPrevResDir :: PerformBuild -> FilePath +pbPrevResDir pb = pbInstallDest pb "prevres" + performBuild :: PerformBuild -> IO [Text] performBuild pb = do cwd <- getWorkingDirectory @@ -161,12 +166,13 @@ performBuild' pb@PerformBuild {..} = withBuildDir $ \builddir -> do $ \ClosedStream Inherited Inherited -> return () let removeTree' fp = whenM (isDirectory fp) (removeTree fp) - mapM_ removeTree' [pbInstallDest, pbLogDir] + removeTree' pbLogDir - forM_ (pbDatabase pb) $ \db -> do - createTree $ parent db - withCheckedProcess (proc "ghc-pkg" ["init", fpToString db]) - $ \ClosedStream Inherited Inherited -> return () + forM_ (pbDatabase pb) $ \db -> + unlessM (isFile $ db "package.cache") $ do + createTree $ parent db + withCheckedProcess (proc "ghc-pkg" ["init", fpToString db]) + $ \ClosedStream Inherited Inherited -> return () pbLog $ encodeUtf8 "Copying built-in Haddocks\n" copyBuiltInHaddocks (pbDocDir pb) @@ -186,7 +192,12 @@ performBuild' pb@PerformBuild {..} = withBuildDir $ \builddir -> do env <- getEnvironment haddockFiles <- newTVarIO mempty - forM_ packageMap $ \pi -> void $ async $ singleBuild pb SingleBuild + registeredPackages <- setupPackageDatabase + (pbDatabase pb) + (ppVersion <$> bpPackages pbPlan) + + forM_ packageMap $ \pi -> void $ async $ singleBuild pb registeredPackages + SingleBuild { sbSem = sem , sbErrsVar = errsVar , sbWarningsVar = warningsVar @@ -248,8 +259,10 @@ data SingleBuild = SingleBuild , sbHaddockFiles :: TVar (Map Text FilePath) -- ^ package-version, .haddock file } -singleBuild :: PerformBuild -> SingleBuild -> IO () -singleBuild pb@PerformBuild {..} SingleBuild {..} = +singleBuild :: PerformBuild + -> Set PackageName -- ^ registered packages + -> SingleBuild -> IO () +singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = withCounter sbActive $ handle updateErrs $ (`finally` void (atomically $ tryPutTMVar (piResult sbPackageInfo) False)) @@ -261,11 +274,12 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = let wfd comps = waitForDeps sbToolMap sbPackageMap comps pbPlan sbPackageInfo . withTSem sbSem - wfd libComps buildLibrary + withUnpacked <- wfd libComps buildLibrary - wfd testComps runTests + wfd testComps (runTests withUnpacked) - name = display $ piName sbPackageInfo + pname = piName sbPackageInfo + name = display pname namever = concat [ name , "-" @@ -333,19 +347,34 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = buildLibrary = wf libOut $ \outH -> do let run a b = do when pbVerbose $ log' (unwords (a : b)) runChild outH a b - log' $ "Unpacking " ++ namever - runParent outH "cabal" ["unpack", namever] - log' $ "Configuring " ++ namever - run "cabal" $ "configure" : configArgs + isUnpacked <- newIORef False + let withUnpacked inner = do + unlessM (readIORef isUnpacked) $ do + log' $ "Unpacking " ++ namever + runParent outH "cabal" ["unpack", namever] + writeIORef isUnpacked True + inner - log' $ "Building " ++ namever - run "cabal" ["build"] + isConfiged <- newIORef False + let withConfiged inner = withUnpacked $ do + unlessM (readIORef isConfiged) $ do + log' $ "Configuring " ++ namever + run "cabal" $ "configure" : configArgs + writeIORef isConfiged True + inner - log' $ "Copying/registering " ++ namever - run "cabal" ["copy"] - withMVar sbRegisterMutex $ const $ - run "cabal" ["register"] + unless (pname `member` registeredPackages) $ withConfiged $ do + deletePreviousResults pb pname + -- FIXME delete old Haddocks? + + log' $ "Building " ++ namever + run "cabal" ["build"] + + log' $ "Copying/registering " ++ namever + run "cabal" ["copy"] + withMVar sbRegisterMutex $ const $ + run "cabal" ["register"] -- Even if the tests later fail, we can allow other libraries to build -- on top of our successful results @@ -355,7 +384,11 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = -- dependency's haddocks before this finishes atomically $ putTMVar (piResult sbPackageInfo) True - when (pbEnableHaddock && pcHaddocks /= Don'tBuild && not (null $ sdModules $ ppDesc $ piPlan sbPackageInfo)) $ do + prevHaddockResult <- getPreviousResult pb Haddock pname + let needHaddock = pbEnableHaddock + && checkPrevResult prevHaddockResult pcHaddocks + && not (null $ sdModules $ ppDesc $ piPlan sbPackageInfo) + when needHaddock $ withConfiged $ do log' $ "Haddocks " ++ namever hfs <- readTVarIO sbHaddockFiles let hfsOpts = flip map (mapToList hfs) $ \(pkgVer, hf) -> concat @@ -390,15 +423,21 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = $ modifyTVar sbHaddockFiles $ insertMap namever newPath + savePreviousResult pb Haddock pname $ either (const False) (const True) eres case (eres, pcHaddocks) of (Left e, ExpectSuccess) -> throwM e (Right (), ExpectFailure) -> warn $ namever ++ ": unexpected Haddock success" _ -> return () - runTests = wf testOut $ \outH -> do + return withUnpacked + + runTests withUnpacked = wf testOut $ \outH -> do let run = runChild outH - when (pbEnableTests && pcTests /= Don'tBuild) $ do + prevTestResult <- getPreviousResult pb Test pname + let needTest = pbEnableTests + && checkPrevResult prevTestResult pcTests + when needTest $ withUnpacked $ do log' $ "Test configure " ++ namever run "cabal" $ "configure" : "--enable-tests" : configArgs @@ -409,6 +448,7 @@ singleBuild pb@PerformBuild {..} SingleBuild {..} = log' $ "Test run " ++ namever run "cabal" ["test", "--log=" ++ fpToText testRunOut] + savePreviousResult pb Test pname $ either (const False) (const True) eres case (eres, pcTests) of (Left e, ExpectSuccess) -> throwM e (Right (), ExpectFailure) -> warn $ namever ++ ": unexpected test success" @@ -451,3 +491,50 @@ copyBuiltInHaddocks docdir = do src <- canonicalizePath (parent (fpFromString ghc) "../share/doc/ghc/html/libraries") copyDir src docdir + +------------- Previous results + +-- | The previous actions that can be run +data ResultType = Haddock | Test + deriving (Show, Enum, Eq, Ord, Bounded, Read) + +-- | The result generated on a previous run +data PrevResult = PRNoResult | PRSuccess | PRFailure + +-- | Check if we should rerun based on a PrevResult and the expected status +checkPrevResult _ Don'tBuild = False +checkPrevResult PRNoResult _ = True +checkPrevResult PRSuccess _ = False +checkPrevResult PRFailure ExpectSuccess = True +checkPrevResult PRFailure _ = False + +withPRPath :: PerformBuild -> ResultType -> PackageName -> (FilePath -> IO a) -> IO a +withPRPath pb rt (PackageName name) inner = do + createTree $ parent fp + inner fp + where + fp = pbPrevResDir pb fpFromString (show rt) fpFromString name + +successBS, failureBS :: ByteString +successBS = "success" +failureBS = "failure" + +getPreviousResult :: PerformBuild -> ResultType -> PackageName -> IO PrevResult +getPreviousResult w x y = withPRPath w x y $ \fp -> do + eres <- tryIO $ readFile fp + return $ case eres of + Right bs + | bs == successBS -> PRSuccess + | bs == failureBS -> PRFailure + _ -> PRNoResult + +savePreviousResult :: PerformBuild -> ResultType -> PackageName -> Bool -> IO () +savePreviousResult pb rt name res = + withPRPath pb rt name $ \fp -> writeFile fp $ + if res then successBS else failureBS + +deletePreviousResults :: PerformBuild -> PackageName -> IO () +deletePreviousResults pb name = + forM_ [minBound..maxBound] $ \rt -> + withPRPath pb rt name $ \fp -> + void $ tryIO $ removeFile fp diff --git a/stackage.cabal b/stackage.cabal index 94b42da9..2dec3030 100644 --- a/stackage.cabal +++ b/stackage.cabal @@ -23,6 +23,7 @@ library Stackage.BuildPlan Stackage.CheckBuildPlan Stackage.UpdateBuildPlan + Stackage.GhcPkg Stackage.GithubPings Stackage.InstallBuild Stackage.PackageDescription @@ -63,6 +64,7 @@ library , streaming-commons >= 0.1.7.1 , semigroups , xml-conduit + , conduit executable stackage default-language: Haskell2010 From f5bd0c777d99734bbb69b54bf98aeaff69eb8fee Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 16:59:57 +0200 Subject: [PATCH 66/90] Add expected test failure network-anonymous-i2p --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 7655b963..1b343254 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1069,6 +1069,9 @@ expected-test-failures: # Problems with linking with system libraries on Ubuntu 12.04 - nettle + # Requires locally running services + - network-anonymous-i2p + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From e75b014b8b207330cd5d487010bf72daafc95e4e Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 17:00:56 +0200 Subject: [PATCH 67/90] Delete old Haddocks when unregistering --- Stackage/GhcPkg.hs | 19 ++++++++++++------- Stackage/PerformBuild.hs | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs index 4985d2f2..c741e96a 100644 --- a/Stackage/GhcPkg.hs +++ b/Stackage/GhcPkg.hs @@ -18,19 +18,21 @@ import qualified Filesystem.Path.CurrentOS as FP import Data.Map (Map) import Data.Version (Version) import Stackage.Prelude +import Filesystem (removeTree) setupPackageDatabase :: Maybe FilePath -- ^ database location, Nothing if using global DB + -> FilePath -- ^ documentation root -> Map PackageName Version -- ^ packages and versions to be installed -> IO (Set PackageName) -- ^ packages remaining in the database after cleanup -setupPackageDatabase mdb toInstall = do +setupPackageDatabase mdb docDir toInstall = do registered1 <- getRegisteredPackages flags - forM_ registered1 $ \(PackageIdentifier name version) -> + forM_ registered1 $ \pi@(PackageIdentifier name version) -> case lookup name toInstall of - Just version' | version /= version' -> unregisterPackage flags name + Just version' | version /= version' -> unregisterPackage docDir flags pi _ -> return () broken <- getBrokenPackages flags - forM_ broken $ \(PackageIdentifier name _) -> unregisterPackage flags name + forM_ broken $ unregisterPackage docDir flags foldMap (\(PackageIdentifier name _) -> singletonSet name) <$> getRegisteredPackages flags where @@ -71,9 +73,12 @@ parsePackageIdent = fmap fst . readP_to_S parse . T.unpack -- | Unregister a package. -unregisterPackage :: [String] -> PackageName -> IO () -unregisterPackage flags ident = do +unregisterPackage :: FilePath -- ^ doc directory + -> [String] -> PackageIdentifier -> IO () +unregisterPackage docDir flags ident@(PackageIdentifier name _) = do void (readProcessWithExitCode "ghc-pkg" - ("unregister": flags ++ ["--force", unpack $ display ident]) + ("unregister": flags ++ ["--force", unpack $ display name]) "") + + void $ tryIO $ removeTree $ docDir fpFromText (display ident) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index fe8dac4a..5597cd63 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -194,6 +194,7 @@ performBuild' pb@PerformBuild {..} = withBuildDir $ \builddir -> do registeredPackages <- setupPackageDatabase (pbDatabase pb) + (pbDocDir pb) (ppVersion <$> bpPackages pbPlan) forM_ packageMap $ \pi -> void $ async $ singleBuild pb registeredPackages @@ -366,7 +367,6 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = unless (pname `member` registeredPackages) $ withConfiged $ do deletePreviousResults pb pname - -- FIXME delete old Haddocks? log' $ "Building " ++ namever run "cabal" ["build"] From 75faf6126bfb566efe5e837c571c2b7a295504ab Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 17:57:43 +0200 Subject: [PATCH 68/90] Add logging when unregistering a package --- Stackage/GhcPkg.hs | 18 ++++++++++++------ Stackage/PerformBuild.hs | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs index c741e96a..f6c3eccf 100644 --- a/Stackage/GhcPkg.hs +++ b/Stackage/GhcPkg.hs @@ -1,7 +1,10 @@ {-# LANGUAGE NoImplicitPrelude #-} +{-# LANGUAGE OverloadedStrings #-} -- | General commands related to ghc-pkg. -module Stackage.GhcPkg where +module Stackage.GhcPkg + ( setupPackageDatabase + ) where import Data.Conduit import qualified Data.Conduit.List as CL @@ -23,16 +26,17 @@ import Filesystem (removeTree) setupPackageDatabase :: Maybe FilePath -- ^ database location, Nothing if using global DB -> FilePath -- ^ documentation root + -> (ByteString -> IO ()) -- ^ logging -> Map PackageName Version -- ^ packages and versions to be installed -> IO (Set PackageName) -- ^ packages remaining in the database after cleanup -setupPackageDatabase mdb docDir toInstall = do +setupPackageDatabase mdb docDir log' toInstall = do registered1 <- getRegisteredPackages flags forM_ registered1 $ \pi@(PackageIdentifier name version) -> case lookup name toInstall of - Just version' | version /= version' -> unregisterPackage docDir flags pi + Just version' | version /= version' -> unregisterPackage log' docDir flags pi _ -> return () broken <- getBrokenPackages flags - forM_ broken $ unregisterPackage docDir flags + forM_ broken $ unregisterPackage log' docDir flags foldMap (\(PackageIdentifier name _) -> singletonSet name) <$> getRegisteredPackages flags where @@ -73,9 +77,11 @@ parsePackageIdent = fmap fst . readP_to_S parse . T.unpack -- | Unregister a package. -unregisterPackage :: FilePath -- ^ doc directory +unregisterPackage :: (ByteString -> IO ()) -- ^ log func + -> FilePath -- ^ doc directory -> [String] -> PackageIdentifier -> IO () -unregisterPackage docDir flags ident@(PackageIdentifier name _) = do +unregisterPackage log' docDir flags ident@(PackageIdentifier name _) = do + log' $ "Unregistering " ++ encodeUtf8 (display ident) void (readProcessWithExitCode "ghc-pkg" ("unregister": flags ++ ["--force", unpack $ display name]) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index 5597cd63..7a627a55 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -195,6 +195,7 @@ performBuild' pb@PerformBuild {..} = withBuildDir $ \builddir -> do registeredPackages <- setupPackageDatabase (pbDatabase pb) (pbDocDir pb) + pbLog (ppVersion <$> bpPackages pbPlan) forM_ packageMap $ \pi -> void $ async $ singleBuild pb registeredPackages From 351c03b8812f291f5625d3f8b6d5b096a44a7fe9 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Fri, 13 Mar 2015 08:39:50 +0200 Subject: [PATCH 69/90] Use PackageIdentifier --- Stackage/GhcPkg.hs | 2 +- Stackage/PerformBuild.hs | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs index f6c3eccf..1e5171d8 100644 --- a/Stackage/GhcPkg.hs +++ b/Stackage/GhcPkg.hs @@ -81,7 +81,7 @@ unregisterPackage :: (ByteString -> IO ()) -- ^ log func -> FilePath -- ^ doc directory -> [String] -> PackageIdentifier -> IO () unregisterPackage log' docDir flags ident@(PackageIdentifier name _) = do - log' $ "Unregistering " ++ encodeUtf8 (display ident) + log' $ "Unregistering " ++ encodeUtf8 (display ident) ++ "\n" void (readProcessWithExitCode "ghc-pkg" ("unregister": flags ++ ["--force", unpack $ display name]) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index 7a627a55..e43d0d5a 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -281,6 +281,7 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = wfd testComps (runTests withUnpacked) pname = piName sbPackageInfo + pident = PackageIdentifier pname (ppVersion $ piPlan sbPackageInfo) name = display pname namever = concat [ name @@ -385,7 +386,7 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = -- dependency's haddocks before this finishes atomically $ putTMVar (piResult sbPackageInfo) True - prevHaddockResult <- getPreviousResult pb Haddock pname + prevHaddockResult <- getPreviousResult pb Haddock pident let needHaddock = pbEnableHaddock && checkPrevResult prevHaddockResult pcHaddocks && not (null $ sdModules $ ppDesc $ piPlan sbPackageInfo) @@ -424,7 +425,7 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = $ modifyTVar sbHaddockFiles $ insertMap namever newPath - savePreviousResult pb Haddock pname $ either (const False) (const True) eres + savePreviousResult pb Haddock pident $ either (const False) (const True) eres case (eres, pcHaddocks) of (Left e, ExpectSuccess) -> throwM e (Right (), ExpectFailure) -> warn $ namever ++ ": unexpected Haddock success" @@ -435,7 +436,7 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = runTests withUnpacked = wf testOut $ \outH -> do let run = runChild outH - prevTestResult <- getPreviousResult pb Test pname + prevTestResult <- getPreviousResult pb Test pident let needTest = pbEnableTests && checkPrevResult prevTestResult pcTests when needTest $ withUnpacked $ do @@ -449,7 +450,7 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = log' $ "Test run " ++ namever run "cabal" ["test", "--log=" ++ fpToText testRunOut] - savePreviousResult pb Test pname $ either (const False) (const True) eres + savePreviousResult pb Test pident $ either (const False) (const True) eres case (eres, pcTests) of (Left e, ExpectSuccess) -> throwM e (Right (), ExpectFailure) -> warn $ namever ++ ": unexpected test success" @@ -501,26 +502,28 @@ data ResultType = Haddock | Test -- | The result generated on a previous run data PrevResult = PRNoResult | PRSuccess | PRFailure + deriving (Show, Enum, Eq, Ord, Bounded, Read) -- | Check if we should rerun based on a PrevResult and the expected status +checkPrevResult :: PrevResult -> TestState -> Bool checkPrevResult _ Don'tBuild = False checkPrevResult PRNoResult _ = True checkPrevResult PRSuccess _ = False checkPrevResult PRFailure ExpectSuccess = True checkPrevResult PRFailure _ = False -withPRPath :: PerformBuild -> ResultType -> PackageName -> (FilePath -> IO a) -> IO a -withPRPath pb rt (PackageName name) inner = do +withPRPath :: PerformBuild -> ResultType -> PackageIdentifier -> (FilePath -> IO a) -> IO a +withPRPath pb rt ident inner = do createTree $ parent fp inner fp where - fp = pbPrevResDir pb fpFromString (show rt) fpFromString name + fp = pbPrevResDir pb fpFromString (show rt) fpFromText (display ident) successBS, failureBS :: ByteString successBS = "success" failureBS = "failure" -getPreviousResult :: PerformBuild -> ResultType -> PackageName -> IO PrevResult +getPreviousResult :: PerformBuild -> ResultType -> PackageIdentifier -> IO PrevResult getPreviousResult w x y = withPRPath w x y $ \fp -> do eres <- tryIO $ readFile fp return $ case eres of @@ -529,12 +532,12 @@ getPreviousResult w x y = withPRPath w x y $ \fp -> do | bs == failureBS -> PRFailure _ -> PRNoResult -savePreviousResult :: PerformBuild -> ResultType -> PackageName -> Bool -> IO () -savePreviousResult pb rt name res = - withPRPath pb rt name $ \fp -> writeFile fp $ +savePreviousResult :: PerformBuild -> ResultType -> PackageIdentifier -> Bool -> IO () +savePreviousResult pb rt ident res = + withPRPath pb rt ident $ \fp -> writeFile fp $ if res then successBS else failureBS -deletePreviousResults :: PerformBuild -> PackageName -> IO () +deletePreviousResults :: PerformBuild -> PackageIdentifier -> IO () deletePreviousResults pb name = forM_ [minBound..maxBound] $ \rt -> withPRPath pb rt name $ \fp -> From cd0e717aab6478b91265512e56336c4441478970 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Fri, 13 Mar 2015 08:40:16 +0200 Subject: [PATCH 70/90] File for store build results --- Stackage/PerformBuild.hs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index e43d0d5a..1aef3af3 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -367,8 +367,10 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = writeIORef isConfiged True inner - unless (pname `member` registeredPackages) $ withConfiged $ do - deletePreviousResults pb pname + prevBuildResult <- getPreviousResult pb Build pident + unless (prevBuildResult == PRSuccess) $ withConfiged $ do + assert (pname `notMember` registeredPackages) $ do + deletePreviousResults pb pident log' $ "Building " ++ namever run "cabal" ["build"] @@ -378,6 +380,8 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = withMVar sbRegisterMutex $ const $ run "cabal" ["register"] + savePreviousResult pb Build pident True + -- Even if the tests later fail, we can allow other libraries to build -- on top of our successful results -- @@ -497,7 +501,7 @@ copyBuiltInHaddocks docdir = do ------------- Previous results -- | The previous actions that can be run -data ResultType = Haddock | Test +data ResultType = Build | Haddock | Test deriving (Show, Enum, Eq, Ord, Bounded, Read) -- | The result generated on a previous run From ca28a53e91bcb7d0cb8d95cb206b438351cad724 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Fri, 13 Mar 2015 08:46:02 +0200 Subject: [PATCH 71/90] Switch default library profiling to true (pinging @manny-fp) --- Stackage/BuildConstraints.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Stackage/BuildConstraints.hs b/Stackage/BuildConstraints.hs index e7aae355..c4ce1ceb 100644 --- a/Stackage/BuildConstraints.hs +++ b/Stackage/BuildConstraints.hs @@ -120,7 +120,7 @@ instance FromJSON PackageConstraints where pcBuildBenchmarks <- o .: "build-benchmarks" pcFlagOverrides <- Map.mapKeysWith const mkFlagName <$> o .: "flags" pcMaintainer <- o .:? "maintainer" - pcEnableLibProfile <- fmap (fromMaybe False) (o .:? "library-profiling") + pcEnableLibProfile <- fmap (fromMaybe True) (o .:? "library-profiling") return PackageConstraints {..} -- | The proposed plan from the requirements provided by contributors. From 19a64410d2f440c3d9f1f4fd746aa6b534013f15 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Thu, 12 Mar 2015 16:59:57 +0200 Subject: [PATCH 72/90] Add expected test failure network-anonymous-i2p --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 7655b963..1b343254 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1069,6 +1069,9 @@ expected-test-failures: # Problems with linking with system libraries on Ubuntu 12.04 - nettle + # Requires locally running services + - network-anonymous-i2p + # Haddocks which are expected to fail. Same concept as expected test failures. expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 From d625c4747bc4e4d9ec77ae1139a21d971aa66ed0 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Fri, 13 Mar 2015 12:33:07 +0200 Subject: [PATCH 73/90] Expect Haddock failure: gtk --- build-constraints.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 1b343254..4036dc18 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -1086,6 +1086,10 @@ expected-haddock-failures: # https://github.com/wereHamster/rethinkdb-client-driver/issues/1 - rethinkdb-client-driver + # Requires build before haddock, which doesn't always happen in incremental + # builds. Could consider special-casing this requirement. + - gtk + # Benchmarks which should not be built. Note that Stackage does *not* generally # build benchmarks. The difference here will be whether dependencies for these # benchmarks are included or not. From 8f74bbee495532162ab3d58225ff0a9661ccc019 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sat, 14 Mar 2015 20:47:14 +0200 Subject: [PATCH 74/90] Delete previous results when unregistering Previously, package install status was determined by checking the package database. However, this didn't work for executables, so I switched it to keep explicit track of build results. I forgot to then clear those results when unregistering a package. Pinging @manny-fp @chrisdone --- Stackage/GhcPkg.hs | 11 +++++++---- Stackage/PerformBuild.hs | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs index 1e5171d8..8f9f9431 100644 --- a/Stackage/GhcPkg.hs +++ b/Stackage/GhcPkg.hs @@ -28,15 +28,16 @@ setupPackageDatabase -> FilePath -- ^ documentation root -> (ByteString -> IO ()) -- ^ logging -> Map PackageName Version -- ^ packages and versions to be installed + -> (PackageIdentifier -> IO ()) -- ^ callback to be used when unregistering a package -> IO (Set PackageName) -- ^ packages remaining in the database after cleanup -setupPackageDatabase mdb docDir log' toInstall = do +setupPackageDatabase mdb docDir log' toInstall onUnregister = do registered1 <- getRegisteredPackages flags forM_ registered1 $ \pi@(PackageIdentifier name version) -> case lookup name toInstall of - Just version' | version /= version' -> unregisterPackage log' docDir flags pi + Just version' | version /= version' -> unregisterPackage log' onUnregister docDir flags pi _ -> return () broken <- getBrokenPackages flags - forM_ broken $ unregisterPackage log' docDir flags + forM_ broken $ unregisterPackage log' onUnregister docDir flags foldMap (\(PackageIdentifier name _) -> singletonSet name) <$> getRegisteredPackages flags where @@ -78,10 +79,12 @@ parsePackageIdent = fmap fst . -- | Unregister a package. unregisterPackage :: (ByteString -> IO ()) -- ^ log func + -> (PackageIdentifier -> IO ()) -- ^ callback to be used when unregistering a package -> FilePath -- ^ doc directory -> [String] -> PackageIdentifier -> IO () -unregisterPackage log' docDir flags ident@(PackageIdentifier name _) = do +unregisterPackage log' onUnregister docDir flags ident@(PackageIdentifier name _) = do log' $ "Unregistering " ++ encodeUtf8 (display ident) ++ "\n" + onUnregister ident void (readProcessWithExitCode "ghc-pkg" ("unregister": flags ++ ["--force", unpack $ display name]) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index d922dab9..02fa14b7 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -198,6 +198,7 @@ performBuild' pb@PerformBuild {..} = withBuildDir $ \builddir -> do (pbDocDir pb) pbLog (ppVersion <$> bpPackages pbPlan) + (deletePreviousResults pb) forM_ packageMap $ \pi -> void $ async $ singleBuild pb registeredPackages SingleBuild From eef9c14d248d2e3ad696078e8843894dbf8a4b7c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 15 Mar 2015 15:29:47 +0200 Subject: [PATCH 75/90] Delete libraries when unregistering Pinging @manny-fp --- Stackage/GhcPkg.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Stackage/GhcPkg.hs b/Stackage/GhcPkg.hs index 8f9f9431..27b7f9e1 100644 --- a/Stackage/GhcPkg.hs +++ b/Stackage/GhcPkg.hs @@ -85,9 +85,20 @@ unregisterPackage :: (ByteString -> IO ()) -- ^ log func unregisterPackage log' onUnregister docDir flags ident@(PackageIdentifier name _) = do log' $ "Unregistering " ++ encodeUtf8 (display ident) ++ "\n" onUnregister ident + + -- Delete libraries + sourceProcessWithConsumer + (proc "ghc-pkg" ("describe" : flags ++ [unpack $ display ident])) + (CT.decodeUtf8 + $= CT.lines + $= CL.mapMaybe parseLibraryDir + $= CL.mapM_ (void . tryIO . removeTree)) + void (readProcessWithExitCode "ghc-pkg" ("unregister": flags ++ ["--force", unpack $ display name]) "") void $ tryIO $ removeTree $ docDir fpFromText (display ident) + where + parseLibraryDir = fmap fpFromText . stripPrefix "library-dirs: " From e80a8d0acfbdc47809cd6418f2aa791751ee916c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 15 Mar 2015 15:46:58 +0200 Subject: [PATCH 76/90] Only create log files when needed Did this for two reasons: 1. Easier to read incremental output this way 2. I believe that, with incremental builds, we were running out of file descriptors in some cases due to so rapidly plowing through all of the packages. I'm not certain this was the source of the errors I was seeing, but given (1), it made sense to try this first. --- Stackage/PerformBuild.hs | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index 02fa14b7..2b333044 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -30,7 +30,7 @@ import Stackage.Prelude hiding (pi) import System.Directory (findExecutable) import System.Environment (getEnvironment) import System.IO (IOMode (WriteMode), - withBinaryFile) + openBinaryFile) import System.IO.Temp (withSystemTempDirectory) data BuildException = BuildException (Map PackageName BuildFailure) [Text] @@ -291,11 +291,12 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = , display $ ppVersion $ piPlan sbPackageInfo ] - runIn wdir outH cmd args = - withCheckedProcess cp $ \ClosedStream UseProvidedHandle UseProvidedHandle -> + runIn wdir getOutH cmd args = do + outH <- getOutH + withCheckedProcess (cp outH) $ \ClosedStream UseProvidedHandle UseProvidedHandle -> (return () :: IO ()) where - cp = (proc (unpack $ asText cmd) (map (unpack . asText) args)) + cp outH = (proc (unpack $ asText cmd) (map (unpack . asText) args)) { cwd = Just $ fpToString wdir , std_out = UseHandle outH , std_err = UseHandle outH @@ -321,8 +322,21 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = testRunOut = pbLogDir fpFromText namever "test-run.out" wf fp inner' = do - createTree $ parent fp - withBinaryFile (fpToString fp) WriteMode inner' + ref <- newIORef Nothing + let cleanup = do + mh <- readIORef ref + forM_ mh hClose + getH = do + mh <- readIORef ref + case mh of + Just h -> return h + Nothing -> mask_ $ do + createTree $ parent fp + h <- openBinaryFile (fpToString fp) WriteMode + writeIORef ref $ Just h + return h + + inner' getH `finally` cleanup configArgs = ($ []) $ execWriter $ do when pbAllowNewer $ tell' "--allow-newer" @@ -350,15 +364,15 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = PackageConstraints {..} = ppConstraints $ piPlan sbPackageInfo - buildLibrary = wf libOut $ \outH -> do + buildLibrary = wf libOut $ \getOutH -> do let run a b = do when pbVerbose $ log' (unwords (a : b)) - runChild outH a b + runChild getOutH a b isUnpacked <- newIORef False let withUnpacked inner = do unlessM (readIORef isUnpacked) $ do log' $ "Unpacking " ++ namever - runParent outH "cabal" ["unpack", namever] + runParent getOutH "cabal" ["unpack", namever] writeIORef isUnpacked True inner @@ -440,8 +454,8 @@ singleBuild pb@PerformBuild {..} registeredPackages SingleBuild {..} = return withUnpacked - runTests withUnpacked = wf testOut $ \outH -> do - let run = runChild outH + runTests withUnpacked = wf testOut $ \getOutH -> do + let run = runChild getOutH prevTestResult <- getPreviousResult pb Test pident let needTest = pbEnableTests From 7dfde3ba49fb60209ccef21e1d03ba633b5fee3f Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 15 Mar 2015 18:51:55 +0200 Subject: [PATCH 77/90] Generate v2 bundle during complete build --- Stackage/CompleteBuild.hs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index 80693dad..c3b26144 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -57,6 +57,8 @@ data Settings = Settings , setArgs :: Text -> UploadBundle -> UploadBundle , postBuild :: IO () , distroName :: Text -- ^ distro name on Hackage + , snapshotType :: SnapshotType + , bundleDest :: FilePath } nightlyPlanFile :: Text -- ^ day @@ -81,6 +83,8 @@ nightlySettings day plan' = Settings , plan = plan' , postBuild = return () , distroName = "Stackage" + , snapshotType = STNightly + , bundleDest = fpFromText $ "stackage-nightly-" ++ tshow day ++ ".bundle" } where slug' = "nightly-" ++ day @@ -142,6 +146,10 @@ getSettings man (LTS bumpType) = do putStrLn "Pushing to Git repository" git ["push"] , distroName = "LTSHaskell" + , snapshotType = + case new of + LTSVer x y -> STLTS x y + , bundleDest = fpFromText $ "stackage-lts-" ++ tshow new ++ ".bundle" } data LTSVer = LTSVer !Int !Int @@ -231,7 +239,16 @@ completeBuild buildType buildFlags = withManager tlsManagerSettings $ \man -> do checkBuildPlan plan putStrLn "Performing build" - performBuild (getPerformBuild buildFlags settings) >>= mapM_ putStrLn + let pb = getPerformBuild buildFlags settings + performBuild pb >>= mapM_ putStrLn + + putStrLn $ "Creating bundle (v2) at: " ++ fpToText bundleDest + createBundleV2 CreateBundleV2 + { cb2Plan = plan + , cb2Type = snapshotType + , cb2DocsDir = pbDocDir pb + , cb2Dest = bundleDest + } when (bfDoUpload buildFlags) $ finallyUpload settings man From fb1d2607bc6fc0b906e657b21f92117f116d2d0d Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 15 Mar 2015 19:15:57 +0200 Subject: [PATCH 78/90] Get rid of superfluous quotes --- Stackage/CompleteBuild.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index c3b26144..b5f41759 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -84,7 +84,7 @@ nightlySettings day plan' = Settings , postBuild = return () , distroName = "Stackage" , snapshotType = STNightly - , bundleDest = fpFromText $ "stackage-nightly-" ++ tshow day ++ ".bundle" + , bundleDest = fpFromText $ "stackage-nightly-" ++ day ++ ".bundle" } where slug' = "nightly-" ++ day From 43639bd6c5cd8a5d1eedec378c1ac305d0f4fd46 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 16 Mar 2015 12:33:29 +0200 Subject: [PATCH 79/90] Remove some expected failures c/o @chrisdone --- build-constraints.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index 4036dc18..527574db 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -966,9 +966,6 @@ expected-test-failures: # https://github.com/BioHaskell/octree/issues/4 - Octree - # https://github.com/goldfirere/th-desugar/issues/12 - - th-desugar - # https://github.com/jmillikin/haskell-filesystem/issues/3 - system-filepath @@ -1077,9 +1074,6 @@ expected-haddock-failures: # https://github.com/acw/bytestring-progress/issues/4 - bytestring-progress - # https://github.com/ekmett/gl/issues/4 - - gl - # https://github.com/leventov/yarr/issues/5 - yarr From 923d655e0900dfc642a7a4b588f52339e6a42e4e Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Mon, 16 Mar 2015 17:17:45 +0200 Subject: [PATCH 80/90] Provide options for v2 uploads --- ChangeLog.md | 2 +- Stackage/CompleteBuild.hs | 87 +++++++++++++++++++++++---------------- Stackage/Upload.hs | 3 ++ app/stackage.hs | 36 +++++++++++++++- stackage.cabal | 5 ++- 5 files changed, 93 insertions(+), 40 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 3d7dd179..2aa9aea0 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.6.0 * Upload bundle V2 stuff diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index b5f41759..5f4735e6 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -8,6 +8,7 @@ module Stackage.CompleteBuild , completeBuild , justCheck , justUploadNightly + , getStackageAuthToken ) where import Control.Concurrent (threadDelay) @@ -39,6 +40,7 @@ data BuildFlags = BuildFlags , bfEnableExecDyn :: !Bool , bfVerbose :: !Bool , bfSkipCheck :: !Bool + , bfUploadV2 :: !Bool } deriving (Show) data BuildType = Nightly | LTS BumpType @@ -251,7 +253,7 @@ completeBuild buildType buildFlags = withManager tlsManagerSettings $ \man -> do } when (bfDoUpload buildFlags) $ - finallyUpload settings man + finallyUpload (bfUploadV2 buildFlags) settings man justUploadNightly :: Text -- ^ nightly date @@ -259,41 +261,63 @@ justUploadNightly justUploadNightly day = do plan <- decodeFileEither (fpToString $ nightlyPlanFile day) >>= either throwM return - withManager tlsManagerSettings $ finallyUpload $ nightlySettings day plan + withManager tlsManagerSettings $ finallyUpload False $ nightlySettings day plan + +getStackageAuthToken :: IO Text +getStackageAuthToken = do + mtoken <- lookupEnv "STACKAGE_AUTH_TOKEN" + case mtoken of + Nothing -> decodeUtf8 <$> readFile "/auth-token" + Just token -> return $ pack token -- | The final part of the complete build process: uploading a bundle, -- docs and a distro to hackage. -finallyUpload :: Settings -> Manager -> IO () -finallyUpload settings@Settings{..} man = do +finallyUpload :: Bool -- ^ use v2 upload + -> Settings -> Manager -> IO () +finallyUpload useV2 settings@Settings{..} man = do putStrLn "Uploading bundle to Stackage Server" - mtoken <- lookupEnv "STACKAGE_AUTH_TOKEN" - token <- - case mtoken of - Nothing -> decodeUtf8 <$> readFile "/auth-token" - Just token -> return $ pack token + token <- getStackageAuthToken - now <- epochTime - let ghcVer = display $ siGhcVersion $ bpSystemInfo plan - (ident, mloc) <- flip uploadBundle man $ setArgs ghcVer def - { ubContents = serverBundle now (title ghcVer) slug plan - , ubAuthToken = token - } - putStrLn $ "New ident: " ++ unSnapshotIdent ident - forM_ mloc $ \loc -> - putStrLn $ "Track progress at: " ++ loc + if useV2 + then do + res <- flip uploadBundleV2 man UploadBundleV2 + { ub2Server = def + , ub2AuthToken = token + , ub2Bundle = bundleDest + } + putStrLn $ "New snapshot available at: " ++ res + else do + now <- epochTime + let ghcVer = display $ siGhcVersion $ bpSystemInfo plan + (ident, mloc) <- flip uploadBundle man $ setArgs ghcVer def + { ubContents = serverBundle now (title ghcVer) slug plan + , ubAuthToken = token + } + putStrLn $ "New ident: " ++ unSnapshotIdent ident + forM_ mloc $ \loc -> + putStrLn $ "Track progress at: " ++ loc + + putStrLn "Uploading docs to Stackage Server" + res1 <- tryAny $ uploadDocs UploadDocs + { udServer = def + , udAuthToken = token + , udDocs = pbDocDir pb + , udSnapshot = ident + } man + putStrLn $ "Doc upload response: " ++ tshow res1 + + putStrLn "Uploading doc map" + tryAny (uploadDocMap UploadDocMap + { udmServer = def + , udmAuthToken = token + , udmSnapshot = ident + , udmDocDir = pbDocDir pb + , udmPlan = plan + } man) >>= print postBuild `catchAny` print - putStrLn "Uploading docs to Stackage Server" - res1 <- uploadDocs UploadDocs - { udServer = def - , udAuthToken = token - , udDocs = pbDocDir pb - , udSnapshot = ident - } man - putStrLn $ "Doc upload response: " ++ tshow res1 - ecreds <- tryIO $ readFile "/hackage-creds" case map encodeUtf8 $ words $ decodeUtf8 $ either (const "") id ecreds of [username, password] -> do @@ -301,14 +325,5 @@ finallyUpload settings@Settings{..} man = do res2 <- uploadHackageDistroNamed distroName plan username password man putStrLn $ "Distro upload response: " ++ tshow res2 _ -> putStrLn "No creds found, skipping Hackage distro upload" - - putStrLn "Uploading doc map" - uploadDocMap UploadDocMap - { udmServer = def - , udmAuthToken = token - , udmSnapshot = ident - , udmDocDir = pbDocDir pb - , udmPlan = plan - } man >>= print where pb = getPerformBuild (error "finallyUpload.buildFlags") settings diff --git a/Stackage/Upload.hs b/Stackage/Upload.hs index 3282bf91..28cdf730 100644 --- a/Stackage/Upload.hs +++ b/Stackage/Upload.hs @@ -16,6 +16,8 @@ module Stackage.Upload , uploadDocMap , uploadBundleV2 , UploadBundleV2 (..) + , def + , unStackageServer ) where import Control.Monad.Writer.Strict (execWriter, tell) @@ -224,6 +226,7 @@ data UploadBundleV2 = UploadBundleV2 uploadBundleV2 :: UploadBundleV2 -> Manager -> IO Text uploadBundleV2 UploadBundleV2 {..} man = IO.withBinaryFile (fpToString ub2Bundle) IO.ReadMode $ \h -> do size <- IO.hFileSize h + putStrLn $ "Bundle size: " ++ tshow size req1 <- parseUrl $ unpack $ unStackageServer ub2Server ++ "/upload2" let req2 = req1 { method = "PUT" diff --git a/app/stackage.hs b/app/stackage.hs index 1b9583c7..c53c4ee4 100644 --- a/app/stackage.hs +++ b/app/stackage.hs @@ -10,7 +10,11 @@ import Options.Applicative import Filesystem.Path.CurrentOS (decodeString) import Paths_stackage (version) import Stackage.CompleteBuild +import Stackage.Upload import Stackage.InstallBuild +import Network.HTTP.Client (withManager) +import Network.HTTP.Client.TLS (tlsManagerSettings) +import qualified Data.Text as T main :: IO () main = @@ -62,7 +66,13 @@ main = installBuild installFlags "install" - "Install a snapshot from an existing build plan"] + "Install a snapshot from an existing build plan" + , cmnd + uploadv2 + uploadv2Flags + "upload2" + "Upload a pre-existing v2 bundle" + ] cmnd exec parse name desc = command name $ @@ -98,7 +108,10 @@ main = help "Output verbose detail about the build steps") <*> switch (long "skip-check" <> - help "Skip the check phase, and pass --allow-newer to cabal configure") + help "Skip the check phase, and pass --allow-newer to cabal configure") <*> + switch + (long "upload-v2" <> + help "Use the V2 upload code") nightlyUploadFlags = fromString <$> strArgument (metavar "DATE" <> @@ -161,3 +174,22 @@ main = switch (long "skip-check" <> help "Skip the check phase, and pass --allow-newer to cabal configure") + + uploadv2 (path, url) = withManager tlsManagerSettings $ \man -> do + token <- getStackageAuthToken + res <- flip uploadBundleV2 man UploadBundleV2 + { ub2AuthToken = token + , ub2Server = fromString url + , ub2Bundle = decodeString path + } + putStrLn $ "New URL: " ++ T.unpack res + + uploadv2Flags = (,) + <$> (strArgument + (metavar "BUNDLE-PATH" <> + help "Bundle path")) + <*> strOption + (long "server-url" <> + metavar "SERVER-URL" <> + showDefault <> value (T.unpack $ unStackageServer def) <> + help "Server to upload bundle to") diff --git a/stackage.cabal b/stackage.cabal index df327327..7efbd149 100644 --- a/stackage.cabal +++ b/stackage.cabal @@ -1,5 +1,5 @@ name: stackage -version: 0.5.2 +version: 0.6.0 synopsis: "Stable Hackage," tools for creating a vetted set of packages from Hackage. description: Please see for a description and documentation. homepage: https://github.com/fpco/stackage @@ -75,6 +75,9 @@ executable stackage , stackage , optparse-applicative >= 0.11 , system-filepath + , http-client + , http-client-tls + , text ghc-options: -rtsopts -threaded -with-rtsopts=-N test-suite spec From ca22965695076fbc053c74d0d7df1fc5e349dcf1 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Mar 2015 10:37:27 +0200 Subject: [PATCH 81/90] Upper bound for #476 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 527574db..bed49edf 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -784,6 +784,9 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 + # https://github.com/fpco/stackage/issues/476 + - vector-space < 0.10 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From b872245bb5c0da650edff3c9dde22066559987d4 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Tue, 17 Mar 2015 10:37:27 +0200 Subject: [PATCH 82/90] Upper bound for #476 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 4036dc18..9bdd0e63 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -784,6 +784,9 @@ packages: # https://github.com/fpco/stackage/issues/467 - lens < 4.8 + # https://github.com/fpco/stackage/issues/476 + - vector-space < 0.10 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 56e1afd3e61502b1b3705e54ec0323f66d5ff4c7 Mon Sep 17 00:00:00 2001 From: "Adam C. Foltzer" Date: Tue, 17 Mar 2015 15:46:49 -0700 Subject: [PATCH 83/90] Add gitrev Hey, I've been using stackage for months now, might as well start contributing ;) --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 9bdd0e63..39b08db0 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -738,6 +738,9 @@ packages: "Oleg Grenrus oleg.grenrus@iki.fi @phadej": - waitra + + "Adam C. Foltzer acfoltzer@galois.com @acfoltzer": + - gitrev "Stackage upper bounds": From 705c6b28a72b415e45ddf742f16419daeec63679 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 08:37:57 +0200 Subject: [PATCH 84/90] Extra #440 upper bound --- build-constraints.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 39b08db0..633134b7 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -770,6 +770,7 @@ packages: # https://github.com/fpco/stackage/issues/440 - th-orphans < 0.9 - file-location < 0.4.7 + - th-desugar < 1.5.1 # https://github.com/fpco/stackage/issues/442 - blaze-builder < 0.4 From 03ac82de687fa2360e4be778f31129e4067a35df Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 08:38:07 +0200 Subject: [PATCH 85/90] Upper bound for #478 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 633134b7..00b9e278 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -791,6 +791,9 @@ packages: # https://github.com/fpco/stackage/issues/476 - vector-space < 0.10 + # https://github.com/fpco/stackage/issues/478 + - linear < 1.18 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 1c71e419b18c047489d4d7d98b60209fa77b544e Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 10:56:51 +0200 Subject: [PATCH 86/90] Fix haddocks --- Stackage/PerformBuild.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Stackage/PerformBuild.hs b/Stackage/PerformBuild.hs index 9ed578b9..41a460de 100644 --- a/Stackage/PerformBuild.hs +++ b/Stackage/PerformBuild.hs @@ -92,7 +92,7 @@ waitForDeps toolMap packageMap activeComps bp pi action = do Nothing | isCoreExe exe -> return () -- https://github.com/jgm/zip-archive/issues/23 - -- | otherwise -> throwSTM $ ToolMissing exe + -- - | otherwise -> throwSTM $ ToolMissing exe | otherwise -> return () Just packages -> ofoldl1' (<|>) packages action From 956060c5d3892ec7f426c4aeea7200729c978c5c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 11:13:47 +0200 Subject: [PATCH 87/90] Version bump --- stackage.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackage.cabal b/stackage.cabal index 7efbd149..d12c172b 100644 --- a/stackage.cabal +++ b/stackage.cabal @@ -1,5 +1,5 @@ name: stackage -version: 0.6.0 +version: 0.6.0.1 synopsis: "Stable Hackage," tools for creating a vetted set of packages from Hackage. description: Please see for a description and documentation. homepage: https://github.com/fpco/stackage From b7a582c18c886319f13e50681266f1f2b82b41b7 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 12:00:23 +0200 Subject: [PATCH 88/90] Remove upper bounds and close #478 --- build-constraints.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-constraints.yaml b/build-constraints.yaml index a0a665dd..92e45a9c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -791,9 +791,6 @@ packages: # https://github.com/fpco/stackage/issues/476 - vector-space < 0.10 - # https://github.com/fpco/stackage/issues/478 - - linear < 1.18 - # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From 52580761e7bd6946b0072a9303047801cc895deb Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 12:22:37 +0200 Subject: [PATCH 89/90] Upper bound for ekmett/linear#70 --- build-constraints.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-constraints.yaml b/build-constraints.yaml index 92e45a9c..8191928a 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -791,6 +791,9 @@ packages: # https://github.com/fpco/stackage/issues/476 - vector-space < 0.10 + # https://github.com/ekmett/linear/issues/70 + - linear < 1.18 + # Package flags are applied to individual packages, and override the values of # global-flags package-flags: From d0138fae7aa50ad7afc1359e94000a44199c014c Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Wed, 18 Mar 2015 15:17:03 +0200 Subject: [PATCH 90/90] V2 upload by default, --server-url option when building --- ChangeLog.md | 4 ++++ Stackage/CompleteBuild.hs | 16 +++++++++++----- Stackage/Upload.hs | 1 + app/stackage.hs | 9 +++++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 2aa9aea0..1dddb87e 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +## 0.6.1 + +* Switch to V2 upload by default + ## 0.6.0 * Upload bundle V2 stuff diff --git a/Stackage/CompleteBuild.hs b/Stackage/CompleteBuild.hs index 5f4735e6..fb1cdde5 100644 --- a/Stackage/CompleteBuild.hs +++ b/Stackage/CompleteBuild.hs @@ -40,7 +40,8 @@ data BuildFlags = BuildFlags , bfEnableExecDyn :: !Bool , bfVerbose :: !Bool , bfSkipCheck :: !Bool - , bfUploadV2 :: !Bool + , bfUploadV1 :: !Bool + , bfServer :: !StackageServer } deriving (Show) data BuildType = Nightly | LTS BumpType @@ -253,7 +254,11 @@ completeBuild buildType buildFlags = withManager tlsManagerSettings $ \man -> do } when (bfDoUpload buildFlags) $ - finallyUpload (bfUploadV2 buildFlags) settings man + finallyUpload + (not $ bfUploadV1 buildFlags) + (bfServer buildFlags) + settings + man justUploadNightly :: Text -- ^ nightly date @@ -261,7 +266,7 @@ justUploadNightly justUploadNightly day = do plan <- decodeFileEither (fpToString $ nightlyPlanFile day) >>= either throwM return - withManager tlsManagerSettings $ finallyUpload False $ nightlySettings day plan + withManager tlsManagerSettings $ finallyUpload False def $ nightlySettings day plan getStackageAuthToken :: IO Text getStackageAuthToken = do @@ -273,8 +278,9 @@ getStackageAuthToken = do -- | The final part of the complete build process: uploading a bundle, -- docs and a distro to hackage. finallyUpload :: Bool -- ^ use v2 upload + -> StackageServer -> Settings -> Manager -> IO () -finallyUpload useV2 settings@Settings{..} man = do +finallyUpload useV2 server settings@Settings{..} man = do putStrLn "Uploading bundle to Stackage Server" token <- getStackageAuthToken @@ -282,7 +288,7 @@ finallyUpload useV2 settings@Settings{..} man = do if useV2 then do res <- flip uploadBundleV2 man UploadBundleV2 - { ub2Server = def + { ub2Server = server , ub2AuthToken = token , ub2Bundle = bundleDest } diff --git a/Stackage/Upload.hs b/Stackage/Upload.hs index 28cdf730..64d21094 100644 --- a/Stackage/Upload.hs +++ b/Stackage/Upload.hs @@ -17,6 +17,7 @@ module Stackage.Upload , uploadBundleV2 , UploadBundleV2 (..) , def + , StackageServer , unStackageServer ) where diff --git a/app/stackage.hs b/app/stackage.hs index c53c4ee4..75e83108 100644 --- a/app/stackage.hs +++ b/app/stackage.hs @@ -110,8 +110,13 @@ main = (long "skip-check" <> help "Skip the check phase, and pass --allow-newer to cabal configure") <*> switch - (long "upload-v2" <> - help "Use the V2 upload code") + (long "upload-v1" <> + help "Use the V1 upload code") <*> + (fmap fromString (strOption + (long "server-url" <> + metavar "SERVER-URL" <> + showDefault <> value (T.unpack $ unStackageServer def) <> + help "Server to upload bundle to"))) nightlyUploadFlags = fromString <$> strArgument (metavar "DATE" <>