stackage-server/src/Foundation.hs
Alexey Kuleshevich f5e147ab97
Integration with Pantry and usage of new stackage-snapshots:
* Moved all extensions into modules that are using them, rather than globally,
  since they mess up ghci session and introduce conflicts among
  packages. Removed those from `.ghci` file as well
* Redesigned the schema to use Pantry and moved it into it's own module
* Switched all of the db and cron related stuff to RIO. Yesod part is
  still on classy-prelude
* Got pantry to update stackage-server database from hackage
* Got import of stackage-snapshots implemented
* Moved some logic from all-cabal-tool
* Switched everything to `PackageNameP`, `VersionP`, etc. from a la Text.
* Fixed haddock, so it now does proper redirects and pipes the docs
  correctly. Also implemented piping of json files from S3 bucket,
  so index-doc.json is also served by stackage-server thus making
  Ctrl+S feature work properly on haddock. Fix for commercialhaskell/stackage#4301
* Import of modules is done through cabal file parsing, which slows
  down the initial import process drastically, but incremental update
  is not a problem.
* Just as with modules, dependencies are also imported from cabal file.
* In general improved type safety by introducing a few data types:
  eg. `ModuleNameP`, `HackageCabalInfo`, and many more.
* Implemented pulling of deprecation map from hackages and storing it in db
* Implementation of forward/backward dependencies within a snapshot only.
* Drastically improved performance of cron import job, by checking which
  snapshots are not up to date
* Implemented pulling haddock list from S3 bucket. Modules that have
  documentation are marked from the availability of actual haddock. This
  process happens concurrently with snapshots loading.
* Rearranged modules a bit:
  * github related functions went into it's own module
  * cron related functions where moved from Database to Cron module
  * Split up some functions to reduce individual complexity
* Parallelized package loading in cron job
* Implemented parsed cabal file caching.
* All queries where reqritten with esqueleto
* Syntactic improvements:
  * Added stylish-haskell config
  * Formatted all imports and extensions with stylish-haskell.
  * Fixed inconsistent indentation across all modules
* Many improvements to the package page as well as few others.
* Reimplemented hoogledb creation.
* Dropped dependency on tar in favor of tar-conduit
* Added cli for stackage-server-cron
* Add cabal sha and size to the package page
* Fixed links in hoogle searches. Improved type safety for a hoogle handler
* satckage-server-cron is customizable with cli arguments

Final adjustments for the new stackage server release:

* Upgrade to lts-13.16.
* Stackage server related code has been merged to pantry. Made the code
  compatible with the newer version pantry
* Added cli '--snapshots-repo'
* Add readme to package page
* Adjust snapshots expected format:
  * Added `publish-time`
  * Removed name `field`
  * `compiler` field is now in the `resolver` field with fallback to
    the root
2019-04-30 17:10:33 +03:00

178 lines
6.8 KiB
Haskell

{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE ViewPatterns #-}
module Foundation where
import ClassyPrelude.Yesod
import Data.WebsiteContent
import Settings
import Settings.StaticFiles
import Stackage.Database
import Text.Blaze
import Text.Hamlet (hamletFile)
import Types
import Yesod.AtomFeed
import Yesod.Core.Types (Logger)
import qualified Yesod.Core.Unsafe as Unsafe
import Yesod.GitRepo
import Yesod.GitRev (GitRev)
import qualified RIO
-- | The site argument for your application. This can be a good place to
-- keep settings and values requiring initialization before your application
-- starts running, such as database connections. Every handler will have
-- access to the data present here.
data App = App
{ appSettings :: !AppSettings
, appStatic :: !Static -- ^ Settings for static file serving.
, appHttpManager :: !Manager
, appLogger :: !Logger
, appLogFunc :: !RIO.LogFunc
, appWebsiteContent :: !(GitRepo WebsiteContent)
, appStackageDatabase :: !StackageDatabase
, appLatestStackMatcher :: !(IO (Text -> Maybe Text))
-- ^ Give a pattern, get a URL
, appHoogleLock :: !(MVar ())
-- ^ Avoid concurrent Hoogle queries, see
-- https://github.com/fpco/stackage-server/issues/172
, appMirrorStatus :: !(IO (Status, WidgetFor App ()))
, appGetHoogleDB :: !(SnapName -> IO (Maybe FilePath))
, appGitRev :: !GitRev
}
instance HasHttpManager App where
getHttpManager = appHttpManager
-- This is where we define all of the routes in our application. For a full
-- explanation of the syntax, please see:
-- http://www.yesodweb.com/book/routing-and-handlers
--
-- Note that this is really half the story; in Application.hs, mkYesodDispatch
-- generates the rest of the code. Please see the linked documentation for an
-- explanation for this split.
mkYesodData "App" $(parseRoutesFile "config/routes")
unsafeHandler :: App -> Handler a -> IO a
unsafeHandler = Unsafe.fakeHandlerGetLogger appLogger
defaultLayoutNoContainer :: Widget -> Handler Html
defaultLayoutNoContainer = defaultLayoutWithContainer False
defaultLayoutWithContainer :: Bool -> Widget -> Handler Html
defaultLayoutWithContainer insideContainer widget = do
mmsg <- getMessage
-- We break up the default layout into two components:
-- default-layout is the contents of the body tag, and
-- default-layout-wrapper is the entire page. Since the final
-- value passed to hamletToRepHtml cannot be a widget, this allows
-- you to use normal widget features in default-layout.
cur <- getCurrentRoute
pc <- widgetToPageContent $ do
$(combineStylesheets 'StaticR
[ css_normalize_css
, css_bootstrap_css
, css_bootstrap_responsive_css
])
$((combineScripts 'StaticR
[ js_jquery_js
, js_bootstrap_js
]))
atomLink FeedR "Recent Stackage snapshots"
$(widgetFile "default-layout")
mcurr <- getCurrentRoute
let notHome = mcurr /= Just HomeR
withUrlRenderer $(hamletFile "templates/default-layout-wrapper.hamlet")
-- Please see the documentation for the Yesod typeclass. There are a number
-- of settings which can be configured by overriding methods here.
instance Yesod App where
approot = ApprootRequest $ \app req ->
case appRoot $ appSettings app of
Nothing -> getApprootText guessApproot app req
Just root -> root
-- Store session data on the client in encrypted cookies,
-- default session idle timeout is 120 minutes
makeSessionBackend _ = return Nothing
defaultLayout = defaultLayoutWithContainer True
{- MSS 2018-06-21 Not worrying about broken cabal-install anymore
-- Ideally we would just have an approot that always includes https, and
-- redirect users from non-SSL to SSL connections. However, cabal-install
-- is broken, and does not support TLS. Therefore, we *don't* force the
-- redirect.
--
-- Nonetheless, we want to keep generated links as https:// links. The
-- problem is that sometimes CORS kicks in and breaks a static resource
-- when loading from a non-secure page. So we have this ugly hack: whenever
-- the destination is a static file, don't include the scheme or hostname.
urlRenderOverride y route@StaticR{} =
Just $ uncurry (joinPath y "") $ renderRoute route
urlRenderOverride _ _ = Nothing
-}
{- Temporarily disable to allow for horizontal scaling
-- This function creates static content files in the static folder
-- and names them based on a hash of their content. This allows
-- expiration dates to be set far in the future without worry of
-- users receiving stale content.
addStaticContent =
addStaticContentExternal minifym genFileName Settings.staticDir (StaticR . flip StaticRoute [])
where
-- Generate a unique filename based on the content itself
genFileName lbs
| development = "autogen-" ++ base64md5 lbs
| otherwise = base64md5 lbs
-}
-- Place Javascript at bottom of the body tag so the rest of the page loads first
jsLoader _ = BottomOfBody
-- What messages should be logged. The following includes all messages when
-- in development, and warnings and errors in production.
shouldLogIO _ "CLEANUP" _ = pure False
shouldLogIO app _source level = pure $
appShouldLogAll (appSettings app)
|| level == LevelWarn
|| level == LevelError
makeLogger = return . appLogger
maximumContentLength _ _ = Just 2000000
instance ToMarkup (Route App) where
toMarkup c =
case c of
AllSnapshotsR{} -> "Snapshots"
BlogHomeR -> "Blog"
_ -> ""
-- This instance is required to use forms. You can modify renderMessage to
-- achieve customized and internationalized form validation messages.
instance RenderMessage App FormMessage where
renderMessage _ _ = defaultFormMessage
-- Note: previous versions of the scaffolding included a deliver function to
-- send emails. Unfortunately, there are too many different options for us to
-- give a reasonable default. Instead, the information is available on the
-- wiki:
--
-- https://github.com/yesodweb/yesod/wiki/Sending-email
instance GetStackageDatabase App Handler where
getStackageDatabase = appStackageDatabase <$> getYesod
getLogFunc = appLogFunc <$> getYesod
instance GetStackageDatabase App (WidgetFor App) where
getStackageDatabase = appStackageDatabase <$> getYesod
getLogFunc = appLogFunc <$> getYesod