From ec239622085a099283ac8b12de3bbbf94e4165c4 Mon Sep 17 00:00:00 2001 From: Michael Snoyman Date: Sun, 5 Apr 2020 15:54:27 +0300 Subject: [PATCH] Move around Dockerfiles --- .azure/azure-linux-template.yml | 21 --- .azure/azure-pipelines.yml | 18 -- .dockerignore | 2 + .github/workflows/base.yml | 5 +- .github/workflows/runtime.yml | 2 +- README.md | 2 +- .../Dockerfile.base-build | 3 +- .../Dockerfile => docker/Dockerfile.base-run | 0 Dockerfile => docker/Dockerfile.runtime | 0 etc/kube/cron_deployment_template.yaml | 61 ------- etc/kube/deployment_template.yaml | 65 ------- etc/kube/hoogle_deployment_template.yaml | 65 ------- etc/kube/hoogle_service_template.yaml | 19 --- etc/kube/ingress_template.yaml | 31 ---- etc/kube/service_template.yaml | 19 --- etc/load-testing/README.md | 30 ---- etc/load-testing/locustfile.py | 159 ------------------ etc/load-testing/requirements.txt | 17 -- etc/scripts/stage_docker.sh | 7 - 19 files changed, 9 insertions(+), 517 deletions(-) delete mode 100644 .azure/azure-linux-template.yml delete mode 100644 .azure/azure-pipelines.yml create mode 100644 .dockerignore rename etc/base-build/Dockerfile => docker/Dockerfile.base-build (67%) rename etc/base-run/Dockerfile => docker/Dockerfile.base-run (100%) rename Dockerfile => docker/Dockerfile.runtime (100%) delete mode 100644 etc/kube/cron_deployment_template.yaml delete mode 100644 etc/kube/deployment_template.yaml delete mode 100644 etc/kube/hoogle_deployment_template.yaml delete mode 100644 etc/kube/hoogle_service_template.yaml delete mode 100644 etc/kube/ingress_template.yaml delete mode 100644 etc/kube/service_template.yaml delete mode 100644 etc/load-testing/README.md delete mode 100644 etc/load-testing/locustfile.py delete mode 100644 etc/load-testing/requirements.txt delete mode 100755 etc/scripts/stage_docker.sh diff --git a/.azure/azure-linux-template.yml b/.azure/azure-linux-template.yml deleted file mode 100644 index 8b83a12..0000000 --- a/.azure/azure-linux-template.yml +++ /dev/null @@ -1,21 +0,0 @@ -jobs: -- job: ${{ parameters.name }} - timeoutInMinutes: 120 - pool: - vmImage: ${{ parameters.vmImage }} - steps: - - script: | - set -ex - export STACK_ROOT="$(Build.SourcesDirectory)"/.stack-root; - mkdir -p ~/.local/bin - curl -L https://get.haskellstack.org/stable/linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack' - export PATH=$HOME/.local/bin:$PATH - - sudo apt-get update - sudo apt-get install libpq-dev - - stack test --interleaved-output - set +ex - env: - OS_NAME: ${{ parameters.os }} - displayName: Installation diff --git a/.azure/azure-pipelines.yml b/.azure/azure-pipelines.yml deleted file mode 100644 index 7c39166..0000000 --- a/.azure/azure-pipelines.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This is the complex Azure configuration, which is intended for use -# on open source libraries which need compatibility across multiple GHC -# versions, must work with cabal-install, and should be -# cross-platform. For more information and other options, see: -# -# https://docs.haskellstack.org/en/stable/azure_ci/ -# -# Copy these contents into the root directory of your Github project in a file -# named azure-pipelines.yml -# -# For better organization, you split various jobs into seprate parts -# and each of them are controlled via individual file. -jobs: -- template: azure-linux-template.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 - os: linux diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..674541b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +.stack-work diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index c2f28e9..52d696a 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -18,13 +18,14 @@ jobs: - name: Build base run image run: | IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/base-run:$GITHUB_SHA - docker build etc/base-run --tag $IMAGE_ID + docker build . -f docker/Dockerfile.base-run --tag base-run + docker tag base-run $IMAGE_ID docker push $IMAGE_ID echo Pushed new base image $IMAGE_ID - name: Build base build image run: | IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/base-build:$GITHUB_SHA - docker build etc/base-build --tag $IMAGE_ID + docker build . -f docker/Dockerfile.base-build --tag $IMAGE_ID docker push $IMAGE_ID echo Pushed new base build image $IMAGE_ID diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml index 3a0c259..f0adc3b 100644 --- a/.github/workflows/runtime.yml +++ b/.github/workflows/runtime.yml @@ -16,7 +16,7 @@ jobs: run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - name: Build image - run: docker build . --tag image + run: docker build . -f docker/Dockerfile.runtime --tag image - name: Push to Docker Hub run: | diff --git a/README.md b/README.md index d3f085e..be3ca04 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ stackage-server =============== -[![Build Status](https://dev.azure.com/fpco/stackage-server/_apis/build/status/fpco.stackage-server?branchName=master)](https://dev.azure.com/fpco/stackage-server/_build/latest?definitionId=1&branchName=master) +![Runtime image](https://github.com/fpco/stackage-server/workflows/Runtime%20image/badge.svg) Server for stable, curated Haskell package sets diff --git a/etc/base-build/Dockerfile b/docker/Dockerfile.base-build similarity index 67% rename from etc/base-build/Dockerfile rename to docker/Dockerfile.base-build index f72bdf4..63ca5bb 100644 --- a/etc/base-build/Dockerfile +++ b/docker/Dockerfile.base-build @@ -1,5 +1,6 @@ -FROM fpco/stack-build:lts-15.5 +FROM base-run RUN stack update COPY stack.yaml stackage-server.cabal /src/ +RUN stack setup --stack-yaml /src/stack.yaml RUN stack build --only-snapshot --stack-yaml /src/stack.yaml diff --git a/etc/base-run/Dockerfile b/docker/Dockerfile.base-run similarity index 100% rename from etc/base-run/Dockerfile rename to docker/Dockerfile.base-run diff --git a/Dockerfile b/docker/Dockerfile.runtime similarity index 100% rename from Dockerfile rename to docker/Dockerfile.runtime diff --git a/etc/kube/cron_deployment_template.yaml b/etc/kube/cron_deployment_template.yaml deleted file mode 100644 index 1c2bfc7..0000000 --- a/etc/kube/cron_deployment_template.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Kubernetes ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: "${CRON_DEPLOYMENT_NAME}" -spec: - replicas: 1 - minReadySeconds: 5 - strategy: - type: Recreate - template: - metadata: - labels: - app: "${CRON_DEPLOYMENT_APP}" - spec: - volumes: - - name: stackage-server-cron-volume - secret: - secretName: ${CRON_DEPLOYMENT_NAME}-secret - containers: - - name: stackage-server-cron - image: "${DEPLOYMENT_IMAGE}" - imagePullPolicy: Always - env: - - name: PGSTRING - valueFrom: - secretKeyRef: - name: "${DEPLOYMENT_NAME}" - key: PGCONN - command: - - bash - - "-c" - - | - set -eux - source /secret/aws.sh - mkdir -p /work - cd /work - while true - do - date - stack update - /usr/local/bin/stackage-server-cron - sleep 5m - done - resources: - # NOTE: This differs between prod and ci. - # Once we migrate to helm we'll be able to set it - # per environment. - requests: - cpu: 150m - memory: 2048Mi - limits: - cpu: 300m - memory: 4096Mi - volumeMounts: - - name: stackage-server-cron-volume - readOnly: true - mountPath: /secret - imagePullSecrets: - - name: registry-key diff --git a/etc/kube/deployment_template.yaml b/etc/kube/deployment_template.yaml deleted file mode 100644 index 37bc781..0000000 --- a/etc/kube/deployment_template.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Kubernetes ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: "${DEPLOYMENT_NAME}" -spec: - replicas: 2 - minReadySeconds: 5 - template: - metadata: - labels: - app: "${DEPLOYMENT_APP}" - spec: - containers: - - name: stackage-server - image: "${DEPLOYMENT_IMAGE}" - imagePullPolicy: Always - ports: - - name: http - containerPort: 3000 - command: ["stackage-server"] - workingDir: /app - env: - - name: APPROOT - value: "${APPROOT}" - - name: PGSTRING - valueFrom: - secretKeyRef: - name: "${DEPLOYMENT_NAME}" - key: PGCONN - readinessProbe: - httpGet: - path: / - port: 3000 - httpHeaders: - # Works around stackage-server's `forceSSL` redirect - - name: HTTPS - value: "on" - initialDelaySeconds: 5 - timeoutSeconds: 1 - periodSeconds: 5 - successThreshold: 1 - livenessProbe: - httpGet: - path: / - port: 3000 - httpHeaders: - # Works around stackage-server's `forceSSL` redirect - - name: HTTPS - value: "on" - initialDelaySeconds: 120 - timeoutSeconds: 1 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - resources: - requests: - cpu: 100m - memory: 1.0Gi - limits: - cpu: 150m - memory: 1.5Gi - imagePullSecrets: - - name: registry-key diff --git a/etc/kube/hoogle_deployment_template.yaml b/etc/kube/hoogle_deployment_template.yaml deleted file mode 100644 index ceadb8f..0000000 --- a/etc/kube/hoogle_deployment_template.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Kubernetes ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: "${HOOGLE_DEPLOYMENT_NAME}" -spec: - replicas: 2 - minReadySeconds: 5 - template: - metadata: - labels: - app: "${HOOGLE_DEPLOYMENT_APP}" - spec: - containers: - - name: stackage-server-hoogle - image: "${DEPLOYMENT_IMAGE}" - imagePullPolicy: Always - ports: - - name: http - containerPort: 3000 - command: ["stackage-server"] - workingDir: /app - env: - - name: APPROOT - value: "${APPROOT}" - - name: PGSTRING - valueFrom: - secretKeyRef: - name: "${DEPLOYMENT_NAME}" - key: PGCONN - readinessProbe: - httpGet: - path: / - port: 3000 - httpHeaders: - # Works around stackage-server's `forceSSL` redirect - - name: HTTPS - value: "on" - initialDelaySeconds: 5 - timeoutSeconds: 1 - periodSeconds: 5 - successThreshold: 1 - livenessProbe: - httpGet: - path: / - port: 3000 - httpHeaders: - # Works around stackage-server's `forceSSL` redirect - - name: HTTPS - value: "on" - initialDelaySeconds: 120 - timeoutSeconds: 1 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - resources: - requests: - cpu: 100m - memory: 1.0Gi - limits: - cpu: 150m - memory: 1.5Gi - imagePullSecrets: - - name: registry-key diff --git a/etc/kube/hoogle_service_template.yaml b/etc/kube/hoogle_service_template.yaml deleted file mode 100644 index bff6359..0000000 --- a/etc/kube/hoogle_service_template.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Kubernetes ---- -apiVersion: v1 -kind: Service -metadata: - name: "${HOOGLE_DEPLOYMENT_NAME}" - labels: - app: "${HOOGLE_DEPLOYMENT_APP}" -spec: - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: http - type: ClusterIP - selector: - app: "${HOOGLE_DEPLOYMENT_APP}" diff --git a/etc/kube/ingress_template.yaml b/etc/kube/ingress_template.yaml deleted file mode 100644 index ba6dca9..0000000 --- a/etc/kube/ingress_template.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: "${DEPLOYMENT_NAME}" - labels: - app: "${DEPLOYMENT_APP}" - annotations: - kubernetes.io/ingress.class: "nginx" - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/use-regex: "true" -spec: - rules: - - host: "${HOST}" - http: - paths: - - path: /haddock.* - backend: - serviceName: "${DEPLOYMENT_NAME}" - servicePort: 80 - - path: /.+/hoogle - backend: - serviceName: "${HOOGLE_DEPLOYMENT_NAME}" - servicePort: 80 - - backend: - serviceName: "${DEPLOYMENT_NAME}" - servicePort: 80 - tls: - - hosts: - - "${HOST}" - secretName: "${DEPLOYMENT_NAME}-tls" - diff --git a/etc/kube/service_template.yaml b/etc/kube/service_template.yaml deleted file mode 100644 index 53dfdf6..0000000 --- a/etc/kube/service_template.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Kubernetes ---- -apiVersion: v1 -kind: Service -metadata: - name: "${DEPLOYMENT_NAME}" - labels: - app: "${DEPLOYMENT_APP}" -spec: - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: http - selector: - app: "${DEPLOYMENT_APP}" - type: ClusterIP diff --git a/etc/load-testing/README.md b/etc/load-testing/README.md deleted file mode 100644 index 8b96c4c..0000000 --- a/etc/load-testing/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# `stackage-server` load tests - -This directory can be used to test an on-premises instance of `stackage-server` for quality of service metrics. Follow the instructions below to run it. - -We assume you will be using `pyenv` to handle different Python environments in a safe manner. If you prefer installing packages globally (not recommended), jump straight to the line commented with `# install dependencies`. Before doing anything, set an environment variable `STACKAGE_SERVER_REPO` on your shell to this repo’s root folder. Then copy everything and run: - -```shell -cd ~ -curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash - -# add the following lines to your shell initialisation for permanent usage -export PYENV_VIRTUALENV_DISABLE_PROMPT=1 -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" - -# create an environment for tests -pyenv install 3.6.3 -pyenv virtualenv 3.6.3 stackage-server-py-3.6.3 -cd "${STACKAGE_SERVER_REPO}/etc/load-testing" -pyenv local stackage-server-py-3.6.3 -pyenv activate stackage-server-py-3.6.3 # should happen automatically on previous line, but just to be sure - -# install dependencies -pip install -r requirements.txt - -# execute load tests -locust --host="http://yourlocalinstance.domain" -``` - -Then navigate to [127.0.0.1:8089](http://127.0.0.1:8089), set the number of clients to simulate, spawn rate, start it and wait a few minutes for the results to stabilise. You can then download or analyse directly on Locust’s UI. diff --git a/etc/load-testing/locustfile.py b/etc/load-testing/locustfile.py deleted file mode 100644 index 0089702..0000000 --- a/etc/load-testing/locustfile.py +++ /dev/null @@ -1,159 +0,0 @@ -from locust import HttpLocust, task, TaskSet -from random import randrange - -def random_element(xs): - return xs[randrange(len(xs))] - -def select_snapshot(): - _snapshots = [ - "lts", - "nightly", - "lts-9.10", - "lts-9.7", - "lts-9.6", - "lts-9.5", - "lts-8.8", - "nightly-2017-07-05", - "nightly-2017-05-30", - "nightly-2017-03-25", - "lts-7.20", - ] - return random_element(_snapshots) - -def select_package(): - _packages = [ - "accelerate", - "adjunctions", - "aeson", - "binary", - "both", - "extensible-effects", - "hamlet", - "hdocs", - "microlens", - "range", - "sort", - "text", - "universe" - ] - return random_element(_packages) - -def select_hoogle_query(): - _hoogle_queries = [ - "Ord", - "Eq", - "Num", - "pack", - "Text -> String", - "fmap", - "a -> a", - "traverse", - "bracket", - "^.", - ">>>", - "<$>", - "bimap", - "inject" - ] - return random_element(_hoogle_queries) - -class HoogleQueries(TaskSet): - @task - def hoogle_queries(self): - _snapshot = select_snapshot() - _query = select_hoogle_query() - self.client.get("/" \ - + _snapshot \ - + "/hoogle?q=" + _query \ - , name="/:snapshot/hoogle?q=[:query]") - - @task - def stop(self): - self.interrupt() - -class Haddock(TaskSet): - @task - def haddock(self): - _snapshot = select_snapshot() - _package = select_package() - self.client.get("/haddock/" \ - + _snapshot + "/" \ - + _package + "/" \ - + "doc-index-All.html" \ - , name="/haddock/:snapshot/:package/doc-index-All.html") - - @task - def stop(self): - self.interrupt() - -class Documentation(TaskSet): - @task - def docs(self): - _snapshot = select_snapshot() - self.client.get("/" \ - + _snapshot \ - + "/docs" \ - , name="/:snapshot/docs") - - @task - def stop(self): - self.interrupt() - -class PackageBrowser(TaskSet): - @task - def browse_package(self): - _snapshot = select_snapshot() - _package = select_package() - self.client.get("/" \ - + _snapshot \ - + "/package/" + _package \ - , name="/:snapshot/package/:package") - - @task - def stop(self): - self.interrupt() - -class Snapshots(TaskSet): - @task - def updateSnapshots(self): - self.client.get("/download/snapshots.json") - - @task - def stop(self): - self.interrupt() - -class TopLevelPages(TaskSet): - @task(20) - def install(self): - self.client.get("/install") - - @task(10) - def lts(self): - self.client.get("/lts") - - @task(5) - def nightly(self): - self.client.get("/nightly") - - @task(2) - def snapshots(self): - self.client.get("/snapshots") - - @task(2) - def stop(self): - self.interrupt() - -class UserBehaviour(TaskSet): - tasks = { - Haddock : 10, - HoogleQueries : 5, - PackageBrowser : 2, - Documentation : 2, - Snapshots : 1, - TopLevelPages : 1, - } - -class WebsiteUser(HttpLocust): - task_set = UserBehaviour - min_wait = 1000 - max_wait = 9000 diff --git a/etc/load-testing/requirements.txt b/etc/load-testing/requirements.txt deleted file mode 100644 index 5b1fa01..0000000 --- a/etc/load-testing/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -certifi==2017.4.17 -chardet==3.0.4 -click==6.7 -Flask==0.12.2 -gevent==1.2.2 -greenlet==0.4.12 -idna==2.5 -itsdangerous==0.24 -Jinja2==2.9.6 --e git://github.com/locustio/locust.git@072d7752552ff32898253fcd5734c3b64995c17e#egg=locustio -MarkupSafe==1.0 -msgpack-python==0.4.8 -pyzmq==15.2.0 -requests==2.18.1 -six==1.10.0 -urllib3==1.21.1 -Werkzeug==0.12.2 diff --git a/etc/scripts/stage_docker.sh b/etc/scripts/stage_docker.sh deleted file mode 100755 index 3b14a14..0000000 --- a/etc/scripts/stage_docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -ex -cd "$(dirname "${BASH_SOURCE[0]}")/../.." -mkdir -p etc/docker/_artifacts -export LANG=C.UTF-8 -stack install --local-bin-path=etc/docker/_artifacts "$@" -rsync -av --delete config static etc/docker/_artifacts/