From b9cbeb27d40dc11bab46c59dfc2336f0a64f9e52 Mon Sep 17 00:00:00 2001 From: Adam Bergmark Date: Sat, 4 Dec 2021 14:46:19 +0100 Subject: [PATCH] commenter: Insert constraints into build-constraints.yaml --- CURATORS.md | 26 ++++-- build-constraints.yaml | 11 ++- etc/commenter/Cargo.lock | 181 ++++++++++++++++++++++++++++++++++++++ etc/commenter/Cargo.toml | 3 +- etc/commenter/src/lib.rs | 124 ++++++++++++++++++++++++++ etc/commenter/src/main.rs | 56 ++++++++++-- 6 files changed, 378 insertions(+), 23 deletions(-) create mode 100644 etc/commenter/src/lib.rs diff --git a/CURATORS.md b/CURATORS.md index 1a2fd6e3..4cbe245f 100644 --- a/CURATORS.md +++ b/CURATORS.md @@ -421,10 +421,10 @@ errors for builds, tests and benchmarks. ### Large scale enabling/disabling of packages -`etc/commenter` is a binary that semi-automates the translation of -`./check` errors into lines that can be copy pasted into -`build-constraints.yaml`. It can only handle bounds issues, -compilation issues still need to be analyzed manually. +`etc/commenter` is a binary that mostly automates the translation of +`./check` errors into lines that go into `build-constraints.yaml`. It +can only handle bounds issues, compilation issues still need to be +handled manually. It disables all offending packages/test suites/benchmarks, so it is only meant to be used when we close bounds issues and want to disable @@ -434,7 +434,7 @@ packages, and when upgrading GHC. This is currently a rust program, You can install the rust toolchain by using [rustup](https://rustup.rs/). -Then `cd etc/commenter && cargo install --locked --path .` +Then `cargo install --locked --path etc/commenter` #### Example usage @@ -455,7 +455,7 @@ testing-feat (GHC 9 bounds issues, Grandfathered dependencies) (not present) dep Now run: ``` -./check 2>&1 >/dev/null | commenter +./check 2>&1 >/dev/null | commenter add ``` You will get this output: @@ -469,19 +469,27 @@ TESTS - dual-tree # tried dual-tree-0.2.3.0, but its *test-suite* requires the disabled package: testing-feat - pipes-fluid # tried pipes-fluid-0.6.0.1, but its *test-suite* requires the disabled package: pipes-misc + +Adding 1 libs, 2 tests, 0 benches to build-constraints.yaml ``` -The lines under LIBS+EXES should be pasted in the shared section (currently called "GHC 9 bounds issues"). +These bounds are added to build-constraints.yaml automatically. -TESTS have a similar section under `skipped-tests`, and BENCHMARKS under `skipped-benchmarks`. +Re-run this command until no more packages are disabled. #### Re-enabling We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This will automatically pick up packages that have been fixed. +``` +commenter clear +./check 2>&1 >/dev/null | commenter add +``` + +Repeat the second command until no updates are made to build-constraints.yaml. + #### Notes -* Please keep these lists sorted as the diffs will be much cleaner when we re-enable packages and re-run this flow * Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build! diff --git a/build-constraints.yaml b/build-constraints.yaml index edbc0106..8a04d07c 100644 --- a/build-constraints.yaml +++ b/build-constraints.yaml @@ -6667,6 +6667,7 @@ packages: - ziptastic-core < 0 # tried ziptastic-core-0.2.0.3, but its *library* does not support: servant-0.18.3 - zydiskell < 0 # tried zydiskell-0.2.0.0, but its *library* does not support: base-4.15.0.0 - zydiskell < 0 # tried zydiskell-0.2.0.0, but its *library* does not support: storable-record-0.0.6 + # End of Library and exe bounds failures "Stackage upper bounds": @@ -7039,11 +7040,9 @@ skipped-tests: # https://github.com/clash-lang/clash-compiler/issues/1622 - clash-prelude - # - # Test bounds issues - # # See "Large scale enabling/disabling of packages" in CURATORS.md for how to manage this section. # + # Test bounds issues - ENIG # tried ENIG-0.0.1.0, but its *test-suite* requires the disabled package: test-framework-th - IPv6DB # tried IPv6DB-0.3.2, but its *test-suite* does not support: hspec-2.8.5 - IPv6DB # tried IPv6DB-0.3.2, but its *test-suite* does not support: http-client-0.7.9 @@ -7263,6 +7262,7 @@ skipped-tests: - wreq # tried wreq-0.5.3.3, but its *test-suite* requires the disabled package: snap-server - xmlhtml # tried xmlhtml-0.2.5.2, but its *test-suite* does not support: hspec-2.8.5 - yesod-static-angular # tried yesod-static-angular-0.1.8, but its *test-suite* does not support: yesod-test-1.6.12 + # End of Test bounds issues # end of skipped-tests @@ -7788,11 +7788,9 @@ skipped-benchmarks: - prettyprinter - prettyprinter-ansi-terminal # https://github.com/commercialhaskell/stackage/issues/5560 - # - # Benchmark bounds issues - # # See "Large scale enabling/disabling of packages" in CURATORS.md for how to manage this section. # + # Benchmark bounds issues - IntervalMap # tried IntervalMap-0.6.1.2, but its *benchmarks* requires the disabled package: SegmentTree - accelerate-bignum # tried accelerate-bignum-0.3.0.0, but its *benchmarks* requires the disabled package: accelerate-io-vector - accelerate-fourier # tried accelerate-fourier-1.0.0.5, but its *benchmarks* does not support: accelerate-llvm-native-1.3.0.0 @@ -7827,6 +7825,7 @@ skipped-benchmarks: - tz # tried tz-0.1.3.5, but its *benchmarks* requires the disabled package: thyme - unicode-transforms # tried unicode-transforms-0.3.8, but its *benchmarks* does not support: path-0.9.1 - xxhash-ffi # tried xxhash-ffi-0.2.0.0, but its *benchmarks* requires the disabled package: xxhash + # End of Benchmark bounds issues # end of skipped-benchmarks diff --git a/etc/commenter/Cargo.lock b/etc/commenter/Cargo.lock index 3b623a76..c6998282 100644 --- a/etc/commenter/Cargo.lock +++ b/etc/commenter/Cargo.lock @@ -11,12 +11,72 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "commenter" version = "0.1.0" dependencies = [ "lazy-regex", "regex", + "structopt", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", ] [[package]] @@ -42,6 +102,18 @@ dependencies = [ "syn", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119" + [[package]] name = "memchr" version = "2.4.0" @@ -54,6 +126,30 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.27" @@ -89,6 +185,36 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "1.0.73" @@ -100,8 +226,63 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "unicode-xid" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/etc/commenter/Cargo.toml b/etc/commenter/Cargo.toml index 87d6f56b..adb15eab 100644 --- a/etc/commenter/Cargo.toml +++ b/etc/commenter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "commenter" -version = "0.1.0" +version = "0.2.0" authors = ["Adam Bergmark "] edition = "2018" @@ -9,3 +9,4 @@ edition = "2018" [dependencies] lazy-regex = "2.2.1" regex = "1.5.4" +structopt = "0.3.25" diff --git a/etc/commenter/src/lib.rs b/etc/commenter/src/lib.rs new file mode 100644 index 00000000..e4ddd2fe --- /dev/null +++ b/etc/commenter/src/lib.rs @@ -0,0 +1,124 @@ +use std::fs::File; +use std::io::{BufRead, BufReader, LineWriter, Lines, Write}; +use std::path::Path; + +pub fn clear() { + handle(|loc, _lines| match loc { + // Add empty array to keep yaml valid + Location::Lib => vec![" []".to_owned()], + Location::Test | Location::Bench => vec![], + }); +} + +pub fn add(lib: Vec, test: Vec, bench: Vec) { + handle(|loc, mut lines| { + lines.extend(match loc { + Location::Lib => lib.clone(), + Location::Test => test.clone(), + Location::Bench => bench.clone(), + }); + lines.sort(); + lines + }) +} + +enum State { + LookingForLibBounds, + ProcessingLibBounds, + LookingForTestBounds, + ProcessingTestBounds, + LookingForBenchBounds, + ProcessingBenchBounds, + Done, +} + +fn handle(f: F) +where + F: Fn(Location, Vec) -> Vec, +{ + let path = "build-constraints.yaml"; + let mut new_lines: Vec = vec![]; + + let mut state = State::LookingForLibBounds; + let mut buf = vec![]; + for line in read_lines(path).map(|s| s.unwrap()) { + match state { + State::LookingForLibBounds => { + if line == r#" "Library and exe bounds failures":"# { + state = State::ProcessingLibBounds; + } + new_lines.push(line); + } + State::ProcessingLibBounds => { + if line == r#" # End of Library and exe bounds failures"# { + new_lines.extend(f(Location::Lib, buf).into_iter()); + buf = vec![]; + new_lines.push(line); + state = State::LookingForTestBounds; + } else { + // Remove empty section + if line != " []" { + buf.push(line); + } + } + } + State::LookingForTestBounds => { + if line == r#" # Test bounds issues"# { + state = State::ProcessingTestBounds; + } + new_lines.push(line); + } + State::ProcessingTestBounds => { + if line == r#" # End of Test bounds issues"# { + new_lines.extend(f(Location::Test, buf).into_iter()); + buf = vec![]; + new_lines.push(line); + state = State::LookingForBenchBounds; + } else { + buf.push(line); + } + } + State::LookingForBenchBounds => { + if line == r#" # Benchmark bounds issues"# { + state = State::ProcessingBenchBounds; + } + new_lines.push(line); + } + State::ProcessingBenchBounds => { + if line == r#" # End of Benchmark bounds issues"# { + new_lines.extend(f(Location::Bench, buf).into_iter()); + buf = vec![]; + new_lines.push(line); + state = State::Done; + } else { + buf.push(line); + } + } + State::Done => { + new_lines.push(line); + } + } + } + + let file = File::create(path).unwrap(); + let mut file = LineWriter::new(file); + + for line in new_lines { + file.write_all((line + "\n").as_bytes()).unwrap(); + } + file.flush().unwrap(); +} + +enum Location { + Lib, + Test, + Bench, +} + +fn read_lines

(filename: P) -> Lines> +where + P: AsRef, +{ + let file = File::open(filename).unwrap(); + BufReader::new(file).lines() +} diff --git a/etc/commenter/src/main.rs b/etc/commenter/src/main.rs index 9a2cb491..282e5afd 100644 --- a/etc/commenter/src/main.rs +++ b/etc/commenter/src/main.rs @@ -1,8 +1,9 @@ -use std::collections::{HashMap}; +use std::collections::HashMap; use std::io::{self, BufRead}; use lazy_regex::regex; use regex::Regex; +use structopt::StructOpt; type H = HashMap>; @@ -12,7 +13,29 @@ enum Header { Missing { package: String }, } +#[derive(Debug, StructOpt)] +#[structopt( + name = "commenter", + about = "Automates generation of bounds in build-constraints.yaml" +)] +enum Opt { + Clear, + Add, +} + fn main() { + let opt = Opt::from_args(); + match opt { + Opt::Clear => clear(), + Opt::Add => add(), + } +} + +fn clear() { + commenter::clear(); +} + +fn add() { let mut lib_exes: H = Default::default(); let mut tests: H = Default::default(); let mut benches: H = Default::default(); @@ -61,12 +84,18 @@ fn main() { } } + let mut auto_lib_exes = vec![]; + let mut auto_tests = vec![]; + let mut auto_benches = vec![]; + if !lib_exes.is_empty() { println!("\nLIBS + EXES\n"); } for (header, packages) in lib_exes { for (package, version, component) in packages { - printer(" ", &package, true, &version, &component, &header); + let s = printer(" ", &package, true, &version, &component, &header); + println!("{}", s); + auto_lib_exes.push(s); } } @@ -75,7 +104,9 @@ fn main() { } for (header, packages) in tests { for (package, version, component) in packages { - printer(" ", &package, false, &version, &component, &header); + let s = printer(" ", &package, false, &version, &component, &header); + println!("{}", s); + auto_tests.push(s); } } @@ -84,9 +115,20 @@ fn main() { } for (header, packages) in benches { for (package, version, component) in packages { - printer(" ", &package, false, &version, &component, &header); + let s = printer(" ", &package, false, &version, &component, &header); + println!("{}", s); + auto_benches.push(s); } } + + println!(); + println!( + "Adding {lib_exes} libs, {tests} tests, {benches} benches to build-constraints.yaml", + lib_exes = auto_lib_exes.len(), + tests = auto_tests.len(), + benches = auto_benches.len() + ); + commenter::add(auto_lib_exes, auto_tests, auto_benches); } fn printer( @@ -96,9 +138,9 @@ fn printer( version: &str, component: &str, header: &Header, -) { +) -> String { let lt0 = if lt0 { " < 0" } else { "" }; - println!( + format!( "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}", indent = indent, package = package, @@ -116,7 +158,7 @@ fn printer( package = package ), }, - ); + ) } fn insert(h: &mut H, header: Header, package: &str, version: &str, component: &str) {