From e1b2afc755c7342c30f575aa2d388d7d1f6b00b3 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 24 Apr 2023 14:07:32 +0000 Subject: [PATCH] chore(ci): fix --- .mailmap | 5 +- flake.nix | 13 +--- nix/frontend/default.nix | 5 +- nix/frontend/node-env.nix | 117 ++++++++++++++++++++++++++---- nix/frontend/node-packages.nix | 23 +++--- nix/uniworx/node-dependencies.nix | 18 +---- nix/uniworx/well-known.nix | 5 +- package-lock.json | 8 +- package-lock.json.license | 2 +- shell.nix | 3 +- 10 files changed, 137 insertions(+), 62 deletions(-) diff --git a/.mailmap b/.mailmap index dc40904f8..2b49f4244 100644 --- a/.mailmap +++ b/.mailmap @@ -1,5 +1,6 @@ -Gregor Kleen -Gregor Kleen +Gregor Kleen +Gregor Kleen +Gregor Kleen Felix Hamann Steffen Jost Sarah Vaupel diff --git a/flake.nix b/flake.nix index ef75fc9ef..aa8459cf1 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,8 @@ -# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen ,Sarah Vaupel +# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen , Sarah Vaupel # # SPDX-License-Identifier: AGPL-3.0-or-later -rec { +{ inputs = { haskell-nix = { url = "github:input-output-hk/haskell.nix"; @@ -82,14 +82,9 @@ rec { url = "git+https://gitlab.ifi.lmu.de/uni2work/haskell/esqueleto.git?ref=uni2work"; flake = false; }; - - # fontawesome-token = { - # url = "path:/etc/fontawesome-token"; - # flake = false; - # }; }; - outputs = { self, nixpkgs, docker-nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachDefaultSystem + outputs = inputs@{ self, nixpkgs, docker-nixpkgs, flake-utils, haskell-nix, ... }: flake-utils.lib.eachDefaultSystem (system: let frontendSource = pkgs.lib.sourceByRegex ./. [ "^(assets|frontend)(/.*)?$" @@ -105,7 +100,7 @@ rec { "^stack\.yaml\.lock$" "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$" "^routes$" - "^testdata(/.*)?$" + "^(assets|app|hlint|load|messages|models|src|templates|test|testdata)(/.*)?$" ]; pkgs = import nixpkgs { diff --git a/nix/frontend/default.nix b/nix/frontend/default.nix index 8acf53226..30650db51 100644 --- a/nix/frontend/default.nix +++ b/nix/frontend/default.nix @@ -1,9 +1,7 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Sarah Vaupel +# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen , Sarah Vaupel # # SPDX-License-Identifier: AGPL-3.0-or-later -# This file has been generated by node2nix 1.11.1. Do not edit! - {pkgs ? import { inherit system; }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}: @@ -17,5 +15,6 @@ let in import ./node-packages.nix { inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit; + inherit (pkgs.stdenv) fetchurlBoot; inherit nodeEnv; } diff --git a/nix/frontend/node-env.nix b/nix/frontend/node-env.nix index 8126d28d6..bc1e36628 100644 --- a/nix/frontend/node-env.nix +++ b/nix/frontend/node-env.nix @@ -1,7 +1,3 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Sarah Vaupel -# -# SPDX-License-Identifier: AGPL-3.0-or-later - # This file originates from node2nix {lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile, writeShellScript}: @@ -169,7 +165,11 @@ let if(process.argv[2] == "development") { replaceDependencies(packageObj.devDependencies); } + else { + packageObj.devDependencies = {}; + } replaceDependencies(packageObj.optionalDependencies); + replaceDependencies(packageObj.peerDependencies); /* Write the fixed package.json file */ fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); @@ -274,7 +274,7 @@ let # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes reconstructPackageLock = writeTextFile { - name = "addintegrityfields.js"; + name = "reconstructpackagelock.js"; text = '' var fs = require('fs'); var path = require('path'); @@ -284,25 +284,43 @@ let var lockObj = { name: packageObj.name, version: packageObj.version, - lockfileVersion: 1, + lockfileVersion: 2, requires: true, + packages: { + "": { + name: packageObj.name, + version: packageObj.version, + license: packageObj.license, + bin: packageObj.bin, + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + } + }, dependencies: {} }; - function augmentPackageJSON(filePath, dependencies) { + function augmentPackageJSON(filePath, packages, dependencies) { var packageJSON = path.join(filePath, "package.json"); if(fs.existsSync(packageJSON)) { var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + packages[filePath] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + }; dependencies[packageObj.name] = { version: packageObj.version, integrity: "sha1-000000000000000000000000000=", dependencies: {} }; - processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies); } } - function processDependencies(dir, dependencies) { + function processDependencies(dir, packages, dependencies) { if(fs.existsSync(dir)) { var files = fs.readdirSync(dir); @@ -318,23 +336,84 @@ let pkgFiles.forEach(function(entry) { if(stats.isDirectory()) { var pkgFilePath = path.join(filePath, entry); - augmentPackageJSON(pkgFilePath, dependencies); + augmentPackageJSON(pkgFilePath, packages, dependencies); } }); } else { - augmentPackageJSON(filePath, dependencies); + augmentPackageJSON(filePath, packages, dependencies); } } }); } } - processDependencies("node_modules", lockObj.dependencies); + processDependencies("node_modules", lockObj.packages, lockObj.dependencies); fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); ''; }; + # Script that links bins defined in package.json to the node_modules bin directory + # NPM does not do this for top-level packages itself anymore as of v7 + linkBinsScript = writeTextFile { + name = "linkbins.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep); + + if(packageObj.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + if(typeof packageObj.bin == "object") { + Object.keys(packageObj.bin).forEach(function(exe) { + if(fs.existsSync(packageObj.bin[exe])) { + console.log("linking bin '" + exe + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin[exe]), + path.join(nodeModules, ".bin", exe) + ); + } + else { + console.log("skipping non-existent bin '" + exe + "'"); + } + }) + } + else { + if(fs.existsSync(packageObj.bin)) { + console.log("linking bin '" + packageObj.bin + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin), + path.join(nodeModules, ".bin", packageObj.name.split("/").pop()) + ); + } + else { + console.log("skipping non-existent bin '" + packageObj.bin + "'"); + } + } + } + else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + fs.readdirSync(packageObj.directories.bin).forEach(function(exe) { + if(fs.existsSync(path.join(packageObj.directories.bin, exe))) { + console.log("linking bin '" + exe + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.directories.bin, exe), + path.join(nodeModules, ".bin", exe) + ); + } + else { + console.log("skipping non-existent bin '" + exe + "'"); + } + }) + } + ''; + }; + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: let forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; @@ -381,13 +460,18 @@ let npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild + runHook postRebuild + if [ "''${dontNpmInstall-}" != "1" ] then # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. rm -f npm-shrinkwrap.json - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install + npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install fi + + # Link executables defined in package.json + node ${linkBinsScript} ''; # Builds and composes an NPM package including all its dependencies @@ -446,12 +530,15 @@ let then ln -s $out/lib/node_modules/.bin $out/bin - # Patch the shebang lines of all the executables + # Fixup all executables ls $out/bin/* | while read i do file="$(readlink -f "$i")" chmod u+rwx "$file" - patchShebangs "$file" + if isScript "$file" + then + sed -i 's/\r$//' "$file" # convert crlf to lf + fi done fi diff --git a/nix/frontend/node-packages.nix b/nix/frontend/node-packages.nix index 3e6a0c925..d985d3771 100644 --- a/nix/frontend/node-packages.nix +++ b/nix/frontend/node-packages.nix @@ -1,10 +1,6 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen ,Sarah Vaupel -# -# SPDX-License-Identifier: AGPL-3.0-or-later - # This file has been generated by node2nix 1.11.1. Do not edit! -{nodeEnv, fetchurl, fetchgit, nix-gitignore, stdenv, lib, globalBuildInputs ? []}: +{nodeEnv, fetchurl, fetchurlBoot, fetchgit, nix-gitignore, stdenv, lib, globalBuildInputs ? []}: let sources = { @@ -1272,9 +1268,9 @@ let name = "_at_fortawesome_slash_fontawesome-pro"; packageName = "@fortawesome/fontawesome-pro"; version = "6.1.1"; - src = fetchurl { + src = fetchurlBoot { url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/6.1.1/fontawesome-pro-6.1.1.tgz"; - sha512 = "0w6GM8sCXNpcBLUz4bx61JvjjoCvfEIz5wBz2KjLNw9qk1F2XiUWuifXobvLbwaA7kqPGBRPo3U8Zw7zyaJ9sA=="; + hash = "sha256-Pq+n8YWCwwQ9B1Etfqs5sypghBMPWPAJFQ49UAlnS18="; }; }; "@humanwhocodes/config-array-0.9.5" = { @@ -6803,6 +6799,15 @@ let sha512 = "sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q=="; }; }; + "nth-check-1.0.2" = { + name = "nth-check"; + packageName = "nth-check"; + version = "1.0.2"; + src = fetchurl { + url = "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz"; + sha512 = "WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg=="; + }; + }; "nth-check-2.0.1" = { name = "nth-check"; packageName = "nth-check"; @@ -10128,7 +10133,7 @@ let args = { name = "uni2work"; packageName = "uni2work"; - version = "26.1.1"; + version = "27.1.6"; src = ../..; dependencies = [ sources."@ampproject/remapping-2.2.0" @@ -10619,7 +10624,7 @@ let dependencies = [ (sources."css-select-1.2.0" // { dependencies = [ - sources."nth-check-2.0.1" + sources."nth-check-1.0.2" ]; }) sources."css-what-2.1.3" diff --git a/nix/uniworx/node-dependencies.nix b/nix/uniworx/node-dependencies.nix index fa88412aa..34bbd4f17 100644 --- a/nix/uniworx/node-dependencies.nix +++ b/nix/uniworx/node-dependencies.nix @@ -1,19 +1,7 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen +# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen # # SPDX-License-Identifier: AGPL-3.0-or-later -{ inputs, ... }: final: prev: { - uniworxNodeDependencies = (prev.callPackage ../frontend {}).nodeDependencies.override (oldArgs: { - dependencies = - let - srcOverrides = { - "tail.datetime" = inputs.tail-DateTime; - "@fortawesome/fontawesome-pro" = prev.fetchurl { - url = "https://npm.fontawesome.com/@fortawesome/fontawesome-pro/-/5.14.0/fontawesome-pro-5.14.0.tgz"; - curlOpts = "-H @${prev.writeText "headers.txt" "Authorization: Bearer ${builtins.readFile inputs.fontawesome-token}"}"; - hash = "sha256-jGvPrTKKL0rCWRZUEnJEmrOdHyQYs3M5709B1hjmFw4="; - }; - }; - in map (dep: dep // { src = srcOverrides."${dep.packageName}" or dep.src; }) oldArgs.dependencies; - }); +{ ... }: final: prev: { + uniworxNodeDependencies = (prev.callPackage ../frontend {}).nodeDependencies; } diff --git a/nix/uniworx/well-known.nix b/nix/uniworx/well-known.nix index adcb6ee3a..d23d3a836 100644 --- a/nix/uniworx/well-known.nix +++ b/nix/uniworx/well-known.nix @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022 Gregor Kleen +# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen # # SPDX-License-Identifier: AGPL-3.0-or-later @@ -21,7 +21,6 @@ ''; outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "18MfdmJX3q826Q4p2F3SnwS2fCjLN0LUpIV/jqUPH4I="; + outputHash = "sha256-zntsIUu13s5doijk3C8ZniYDlDdHZ5Isbv0SQLNgt08="; }; } diff --git a/package-lock.json b/package-lock.json index 4e34e05fa..0a2b70581 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3829,12 +3829,12 @@ }, "dependencies": { "nth-check": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.1.tgz", - "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", "dev": true, "requires": { - "boolbase": "^1.0.0" + "boolbase": "~1.0.0" } } } diff --git a/package-lock.json.license b/package-lock.json.license index f15f3f211..8e5638ee4 100644 --- a/package-lock.json.license +++ b/package-lock.json.license @@ -1,3 +1,3 @@ -SPDX-FileCopyrightText: 2022 Felix Hamann ,Gregor Kleen ,Sarah Vaupel ,Sarah Vaupel ,Steffen Jost +SPDX-FileCopyrightText: 2022-2023 Felix Hamann , Gregor Kleen , Sarah Vaupel , Steffen Jost SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/shell.nix b/shell.nix index c36f0e8fd..fefbd66c3 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen ,Sarah Vaupel ,Steffen Jost +# SPDX-FileCopyrightText: 2022-2023 Gregor Kleen , Sarah Vaupel , Steffen Jost # # SPDX-License-Identifier: AGPL-3.0-or-later @@ -273,6 +273,7 @@ in pkgs.mkShell { ++ (with pkgs; [ nodejs-14_x postgresql_12 openldap chromium exiftool memcached minio minio-client gup reuse pre-commit + node2nix # busybox # for print services, but interferes with build commands in develop-shell htop pdftk # pdftk just for testing pdf-passwords