chore(ci): fix

2023-04-24 14:07:32 +00:00 · 2023-04-24 14:07:32 +00:00 · e1b2afc755
commit e1b2afc755
parent 4b5d3df141
10 changed files with 137 additions and 62 deletions
--- a/.mailmap
+++ b/.mailmap
@ -1,5 +1,6 @@
-Gregor Kleen <gregor.kleen@ifi.lmu.de> <kleen@cip.ifi.lmu.de>
-Gregor Kleen <gregor.kleen@ifi.lmu.de> <gkleen@yggdrasil.li>
+Gregor Kleen <gregor@kleen.consulting> <gregor.kleen@ifi.lmu.de>
+Gregor Kleen <gregor@kleen.consulting> <kleen@cip.ifi.lmu.de>
+Gregor Kleen <gregor@kleen.consulting> <gkleen@yggdrasil.li>
 Felix Hamann <felix.hamann@campus.lmu.de> <felix.hamann@satellytes.com>
 Steffen Jost <jost@tcs.ifi.lmu.de> <jost@tcs.ifi.lmu.de>
 Sarah Vaupel <vaupel.sarah@campus.lmu.de> <vaupel@cip.ifi.lmu.de>
--- a/flake.nix
+++ b/flake.nix
@ -1,8 +1,8 @@
-# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@uniworx.de>
+# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor@kleen.consulting>, Sarah Vaupel <sarah.vaupel@uniworx.de>
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

-rec {
+{
  inputs = {
    haskell-nix = {
      url = "github:input-output-hk/haskell.nix";
@ -82,14 +82,9 @@ rec {
      url = "git+https://gitlab.ifi.lmu.de/uni2work/haskell/esqueleto.git?ref=uni2work";
      flake = false;
    };
-
-    # fontawesome-token = {
-    #   url = "path:/etc/fontawesome-token";
-    #   flake = false;
-    # };
  };

-  outputs = { self, nixpkgs, docker-nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachDefaultSystem
+  outputs = inputs@{ self, nixpkgs, docker-nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachDefaultSystem
    (system:
      let frontendSource = pkgs.lib.sourceByRegex ./. [
            "^(assets|frontend)(/.*)?$"
@ -105,7 +100,7 @@ rec {
            "^stack\.yaml\.lock$"
            "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$"
            "^routes$"
-            "^testdata(/.*)?$"
+            "^(assets|app|hlint|load|messages|models|src|templates|test|testdata)(/.*)?$"
          ];

          pkgs = import nixpkgs {
--- a/nix/frontend/default.nix
+++ b/nix/frontend/default.nix
@ -1,9 +1,7 @@
-# SPDX-FileCopyrightText: 2022 Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@ifi.lmu.de>
+# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor@kleen.consulting>, Sarah Vaupel <sarah.vaupel@ifi.lmu.de>
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

-# This file has been generated by node2nix 1.11.1. Do not edit!
-
 {pkgs ? import <nixpkgs> {
    inherit system;
  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}:
@ -17,5 +15,6 @@ let
 in
 import ./node-packages.nix {
  inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit;
+  inherit (pkgs.stdenv) fetchurlBoot;
  inherit nodeEnv;
 }
--- a/nix/frontend/node-env.nix
+++ b/nix/frontend/node-env.nix
@ -1,7 +1,3 @@
-# SPDX-FileCopyrightText: 2022 Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@ifi.lmu.de>
-#
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
 # This file originates from node2nix

 {lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile, writeShellScript}:
@ -169,7 +165,11 @@ let
          if(process.argv[2] == "development") {
              replaceDependencies(packageObj.devDependencies);
          }
+          else {
+              packageObj.devDependencies = {};
+          }
          replaceDependencies(packageObj.optionalDependencies);
+          replaceDependencies(packageObj.peerDependencies);

          /* Write the fixed package.json file */
          fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
@ -274,7 +274,7 @@ let

  # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
  reconstructPackageLock = writeTextFile {
-    name = "addintegrityfields.js";
+    name = "reconstructpackagelock.js";
    text = ''
      var fs = require('fs');
      var path = require('path');
@ -284,25 +284,43 @@ let
      var lockObj = {
          name: packageObj.name,
          version: packageObj.version,
-          lockfileVersion: 1,
+          lockfileVersion: 2,
          requires: true,
+          packages: {
+              "": {
+                  name: packageObj.name,
+                  version: packageObj.version,
+                  license: packageObj.license,
+                  bin: packageObj.bin,
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              }
+          },
          dependencies: {}
      };

-      function augmentPackageJSON(filePath, dependencies) {
+      function augmentPackageJSON(filePath, packages, dependencies) {
          var packageJSON = path.join(filePath, "package.json");
          if(fs.existsSync(packageJSON)) {
              var packageObj = JSON.parse(fs.readFileSync(packageJSON));
+              packages[filePath] = {
+                  version: packageObj.version,
+                  integrity: "sha1-000000000000000000000000000=",
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              };
              dependencies[packageObj.name] = {
                  version: packageObj.version,
                  integrity: "sha1-000000000000000000000000000=",
                  dependencies: {}
              };
-              processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
+              processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
          }
      }

-      function processDependencies(dir, dependencies) {
+      function processDependencies(dir, packages, dependencies) {
          if(fs.existsSync(dir)) {
              var files = fs.readdirSync(dir);

@ -318,23 +336,84 @@ let
                          pkgFiles.forEach(function(entry) {
                              if(stats.isDirectory()) {
                                  var pkgFilePath = path.join(filePath, entry);
-                                  augmentPackageJSON(pkgFilePath, dependencies);
+                                  augmentPackageJSON(pkgFilePath, packages, dependencies);
                              }
                          });
                      } else {
-                          augmentPackageJSON(filePath, dependencies);
+                          augmentPackageJSON(filePath, packages, dependencies);
                      }
                  }
              });
          }
      }

-      processDependencies("node_modules", lockObj.dependencies);
+      processDependencies("node_modules", lockObj.packages, lockObj.dependencies);

      fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
    '';
  };

+  # Script that links bins defined in package.json to the node_modules bin directory
+  # NPM does not do this for top-level packages itself anymore as of v7
+  linkBinsScript = writeTextFile {
+    name = "linkbins.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      var packageObj = JSON.parse(fs.readFileSync("package.json"));
+
+      var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
+
+      if(packageObj.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          if(typeof packageObj.bin == "object") {
+              Object.keys(packageObj.bin).forEach(function(exe) {
+                  if(fs.existsSync(packageObj.bin[exe])) {
+                      console.log("linking bin '" + exe + "'");
+                      fs.symlinkSync(
+                          path.join("..", packageObj.name, packageObj.bin[exe]),
+                          path.join(nodeModules, ".bin", exe)
+                      );
+                  }
+                  else {
+                      console.log("skipping non-existent bin '" + exe + "'");
+                  }
+              })
+          }
+          else {
+              if(fs.existsSync(packageObj.bin)) {
+                  console.log("linking bin '" + packageObj.bin + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.bin),
+                      path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + packageObj.bin + "'");
+              }
+          }
+      }
+      else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
+              if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
+                  console.log("linking bin '" + exe + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.directories.bin, exe),
+                      path.join(nodeModules, ".bin", exe)
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + exe + "'");
+              }
+          })
+      }
+    '';
+  };
+
  prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
    let
      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
@ -381,13 +460,18 @@ let

        npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild

+        runHook postRebuild
+
        if [ "''${dontNpmInstall-}" != "1" ]
        then
            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
            rm -f npm-shrinkwrap.json

-            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
        fi
+
+        # Link executables defined in package.json
+        node ${linkBinsScript}
    '';

  # Builds and composes an NPM package including all its dependencies
@ -446,12 +530,15 @@ let
        then
            ln -s $out/lib/node_modules/.bin $out/bin

-            # Patch the shebang lines of all the executables
+            # Fixup all executables
            ls $out/bin/* | while read i
            do
                file="$(readlink -f "$i")"
                chmod u+rwx "$file"
-                patchShebangs "$file"
+                if isScript "$file"
+                then
+                    sed -i 's/\r$//' "$file"  # convert crlf to lf
+                fi
            done
        fi

--- a/nix/frontend/node-packages.nix
+++ b/nix/frontend/node-packages.nix
@ -1,10 +1,6 @@
-# SPDX-FileCopyrightText: 2022 Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@ifi.lmu.de>
-#
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
 # This file has been generated by node2nix 1.11.1. Do not edit!

-{nodeEnv, fetchurl, fetchgit, nix-gitignore, stdenv, lib, globalBuildInputs ? []}:
+{nodeEnv, fetchurl, fetchurlBoot, fetchgit, nix-gitignore, stdenv, lib, globalBuildInputs ? []}:

 let
  sources = {
@ -1272,9 +1268,9 @@ let
      name = "_at_fortawesome_slash_fontawesome-pro";
      packageName = "@fortawesome/fontawesome-pro";
      version = "6.1.1";
-      src = fetchurl {
+      src = fetchurlBoot {
        url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/6.1.1/fontawesome-pro-6.1.1.tgz";
-        sha512 = "0w6GM8sCXNpcBLUz4bx61JvjjoCvfEIz5wBz2KjLNw9qk1F2XiUWuifXobvLbwaA7kqPGBRPo3U8Zw7zyaJ9sA==";
+        hash = "sha256-Pq+n8YWCwwQ9B1Etfqs5sypghBMPWPAJFQ49UAlnS18=";
      };
    };
    "@humanwhocodes/config-array-0.9.5" = {
@ -6803,6 +6799,15 @@ let
        sha512 = "sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==";
      };
    };
+    "nth-check-1.0.2" = {
+      name = "nth-check";
+      packageName = "nth-check";
+      version = "1.0.2";
+      src = fetchurl {
+        url = "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz";
+        sha512 = "WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==";
+      };
+    };
    "nth-check-2.0.1" = {
      name = "nth-check";
      packageName = "nth-check";
@ -10128,7 +10133,7 @@ let
  args = {
    name = "uni2work";
    packageName = "uni2work";
-    version = "26.1.1";
+    version = "27.1.6";
    src = ../..;
    dependencies = [
      sources."@ampproject/remapping-2.2.0"
@ -10619,7 +10624,7 @@ let
        dependencies = [
          (sources."css-select-1.2.0" // {
            dependencies = [
-              sources."nth-check-2.0.1"
+              sources."nth-check-1.0.2"
            ];
          })
          sources."css-what-2.1.3"
--- a/nix/uniworx/node-dependencies.nix
+++ b/nix/uniworx/node-dependencies.nix
@ -1,19 +1,7 @@
-# SPDX-FileCopyrightText: 2022 Gregor Kleen <gregor.kleen@ifi.lmu.de>
+# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor@kleen.consulting>
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

-{ inputs, ... }: final: prev: {
-  uniworxNodeDependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: {
-    dependencies =
-      let
-        srcOverrides = {
-          "tail.datetime" = inputs.tail-DateTime;
-          "@fortawesome/fontawesome-pro" = prev.fetchurl {
-            url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz";
-            curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}";
-            hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4=";
-          };
-        };
-      in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies;
-  });
+{ ... }: final: prev: {
+  uniworxNodeDependencies = (prev.callPackage ../frontend {}).nodeDependencies;
 }
--- a/nix/uniworx/well-known.nix
+++ b/nix/uniworx/well-known.nix
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022 Gregor Kleen <gregor.kleen@ifi.lmu.de>
+# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor@kleen.consulting>
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

@ -21,7 +21,6 @@
    '';

    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
-    outputHash = "18MfdmJX3q826Q4p2F3SnwS2fCjLN0LUpIV/jqUPH4I=";
+    outputHash = "sha256-zntsIUu13s5doijk3C8ZniYDlDdHZ5Isbv0SQLNgt08=";
  };
 }
--- a/package-lock.json
+++ b/package-lock.json
@ -3829,12 +3829,12 @@
          },
          "dependencies": {
            "nth-check": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.1.tgz",
-              "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==",
+              "version": "1.0.2",
+              "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+              "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
              "dev": true,
              "requires": {
-                "boolbase": "^1.0.0"
+                "boolbase": "~1.0.0"
              }
            }
          }
--- a/package-lock.json.license
+++ b/package-lock.json.license
@ -1,3 +1,3 @@
-SPDX-FileCopyrightText: 2022 Felix Hamann <felix.hamann@campus.lmu.de>,Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@ifi.lmu.de>,Sarah Vaupel <vaupel.sarah@campus.lmu.de>,Steffen Jost <jost@tcs.ifi.lmu.de>
+SPDX-FileCopyrightText: 2022-2023 Felix Hamann <felix.hamann@campus.lmu.de>, Gregor Kleen <gregor@kleen.consulting>, Sarah Vaupel <sarah.vaupel@ifi.lmu.de>, Steffen Jost <jost@tcs.ifi.lmu.de>

 SPDX-License-Identifier: AGPL-3.0-or-later
--- a/shell.nix
+++ b/shell.nix
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor.kleen@ifi.lmu.de>,Sarah Vaupel <sarah.vaupel@uniworx.de>,Steffen Jost <jost@tcs.ifi.lmu.de>
+# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen <gregor@kleen.consulting>, Sarah Vaupel <sarah.vaupel@uniworx.de>, Steffen Jost <jost@tcs.ifi.lmu.de>
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

@ -273,6 +273,7 @@ in pkgs.mkShell {
  ++ (with pkgs;
    [ nodejs-14_x postgresql_12 openldap chromium exiftool memcached minio minio-client
      gup reuse pre-commit
+      node2nix
      # busybox # for print services, but interferes with build commands in develop-shell
      htop
      pdftk # pdftk just for testing pdf-passwords