Warn and continue if orig.tar is 404

src/Stackage/Database/Cron.hs

@@ -858,44 +858,57 @@ createHoogleDB rootDir snapshotId snapName = do
         outname = root </> "output.hoo"
         inputTarKey = toPathPiece snapName <> "/hoogle/orig.tar"
         inputTarUrl = downloadBucketUrl <> "/" <> inputTarKey
-        outputTarFP = root </> T.unpack inputTarKey
     -- Fetch the tarball with Hoogle inputs
     req <- parseRequest $ T.unpack inputTarUrl
     env <- asks scEnvAWS
     let man = env ^. env_manager
-    withResponseUnliftIO req {decompress = const True} man $ \res -> do
+    mTarFile <- withResponseUnliftIO req {decompress = const True} man $ \res -> do
         -- FIXME: Catch HttpExceptionRequest and give up on this snapshot?
-        throwErrorStatusCodes req res
+        case responseStatus res of
-        createDirectoryIfMissing True $ takeDirectory outputTarFP
+          status
-        withBinaryFileDurableAtomic outputTarFP WriteMode $ \tarHandle ->
+            | status == status200 -> do
-            runConduitRes $ bodyReaderSource (responseBody res) .| sinkHandle tarHandle
+              let outputTarFP = root </> T.unpack inputTarKey
+              createDirectoryIfMissing True $ takeDirectory outputTarFP
+              withBinaryFileDurableAtomic outputTarFP WriteMode $ \tarHandle ->
+                  runConduitRes $ bodyReaderSource (responseBody res) .| sinkHandle tarHandle
+              pure $ Just outputTarFP
+            | status == status404 -> do
+                  logWarn $ "Input orig.tar is a 404 for " <> display snapName
+                  pure Nothing
+            | otherwise ->
+                  -- NOW we give up.
+                  Nothing <$ throwErrorStatusCodes req res
+
     -- Extract the Hoogle inputs from the tarball into a separate temp dir, then
     -- generate the hoogle database.
-    withSystemTempDirectory ("hoogle-" ++ T.unpack (textDisplay snapName)) $ \tmpdir -> do
+    case mTarFile of
-        Any hasRestored <-
+        Nothing -> pure Nothing
-            runConduitRes $
+        Just outputTarFP -> do
-            sourceFile outputTarFP .|
+            withSystemTempDirectory ("hoogle-" ++ T.unpack (textDisplay snapName)) $ \tmpdir -> do
-            untar (restoreHoogleTxtFileWithCabal tmpdir snapshotId snapName) .|
+                Any hasRestored <-
-            foldMapC Any
+                    runConduitRes $
-        -- We just check if we have any Hoogle .txt file at all.
+                    sourceFile outputTarFP .|
-        -- If there are none, we just give up
+                    untar (restoreHoogleTxtFileWithCabal tmpdir snapshotId snapName) .|
-        if hasRestored then do
+                    foldMapC Any
-            -- Generate the hoogle database
+                -- We just check if we have any Hoogle .txt file at all.
-            let args = ["generate", "--database=" ++ outname, "--local=" ++ tmpdir]
+                -- If there are none, we just give up
-            logInfo $
+                if hasRestored then do
-                mconcat
+                    -- Generate the hoogle database
-                    [ "Merging databases... ("
+                    let args = ["generate", "--database=" ++ outname, "--local=" ++ tmpdir]
-                    , foldMap fromString $ L.intersperse " " ("hoogle" : args)
+                    logInfo $
-                    , ")"
+                        mconcat
-                    ]
+                            [ "Merging databases... ("
-            -- 'Hoogle.hoogle' expects to run as an app, and crashes if something
+                            , foldMap fromString $ L.intersperse " " ("hoogle" : args)
-            -- goes wrong. That's good.
+                            , ")"
-            liftIO $ Hoogle.hoogle args
+                            ]
-            logInfo "Merge done"
+                    -- 'Hoogle.hoogle' expects to run as an app, and crashes if something
-            pure $ Just outname
+                    -- goes wrong. That's good.
-        else do
+                    liftIO $ Hoogle.hoogle args
-            logWarn $ "No Hoogle.txt files found for " <> display snapName <> ", skipping Hoogle DB creation."
+                    logInfo "Merge done"
-            pure Nothing
+                    pure $ Just outname
+                else do
+                    logWarn $ "No Hoogle.txt files found for " <> display snapName <> ", skipping Hoogle DB creation."
+                    pure Nothing
 
 
 -- | Grabs hoogle txt file from the tarball and a matching cabal file from pantry.  Writes