From 2f4c89cd3133517f58bc9e42f29220a1bbb66bcc Mon Sep 17 00:00:00 2001 From: Owen Jacobson Date: Thu, 18 Jun 2020 02:19:43 -0400 Subject: Switch from quickcheck to proptest. The argument is as given in the proptest docs at . I've found that the resulting tests are somewhat clearer, and that the tools for working with test case generation are more useful. The other killer feature is recalling test failure examples from run to run. This change includes at least one bug found while testing the port! Finally, if is to be believed, proptest is considerably closer to supporting async tests. --- Cargo.lock | 239 ++++++++++++++++++++++++++++------------ Cargo.toml | 4 +- proptest-regressions/twelve.txt | 9 ++ src/twelve.rs | 127 ++++++++++----------- 4 files changed, 242 insertions(+), 137 deletions(-) create mode 100644 proptest-regressions/twelve.txt diff --git a/Cargo.lock b/Cargo.lock index cee3a90..f14b40c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,8 +87,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a60f9ba7c4e6df97f3aacb14bb5c0cd7d98a49dcbaed0d7f292912ad9a6a3ed2" dependencies = [ - "quote", - "syn", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -252,9 +252,9 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a71bf475cbe07281d0b3696abb48212db118e7e23219f13596ce865235ff5766" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -293,9 +293,9 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26c4f3195085c36ea8d24d32b2f828d23296a9370a28aa39d111f6f16bef9f3b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -346,6 +346,21 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0dc55f2d8a1a85650ac47858bb001b4c0dd73d79e3c455a842925e68d29cd3" + [[package]] name = "bitflags" version = "1.2.1" @@ -446,9 +461,9 @@ version = "0.99.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2127768764f1556535c01b5326ef94bd60ff08dcfbdc544d53e69ed155610f5d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -479,19 +494,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc4bfcfacb61d231109d1d55202c1f33263319668b168843e02ad4652725ec9c" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "log", - "regex", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -510,9 +515,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", "synstructure", ] @@ -605,9 +610,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -888,7 +893,7 @@ source = "git+https://github.com/lambda-fairy/maud#240ecf03b8c16021a609397a056d6 dependencies = [ "matches", "maud_htmlescape", - "syn", + "syn 1.0.30", ] [[package]] @@ -1051,9 +1056,9 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e58db2081ba5b4c93bd6be09c40fd36cb9193a8336c384f3b40012e531aa7e40" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -1086,13 +1091,53 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.0", +] + +[[package]] +name = "proptest" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2520fe6373cf6a3a61e2d200e987c183778ade8d9248ac3e6614ab0edfe4a0c1" +dependencies = [ + "bit-set", + "bitflags", + "byteorder", + "lazy_static", + "num-traits", + "quick-error", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", +] + +[[package]] +name = "proptest-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051d9d20dbe9e9dfe594328b6eaab72ccf571fee818991dd1c1ba5469b5f32d4" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", ] [[package]] @@ -1114,26 +1159,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] -name = "quickcheck" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" -dependencies = [ - "env_logger", - "log", - "rand", - "rand_core", -] - -[[package]] -name = "quickcheck_macros" -version = "0.9.1" +name = "quote" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608c156fd8e97febc07dc9c2e2c80bf74cfc6ef26893eae3daf8bc2bc94a4b7f" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 0.4.30", ] [[package]] @@ -1142,7 +1173,7 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.18", ] [[package]] @@ -1186,6 +1217,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core", +] + [[package]] name = "redox_syscall" version = "0.1.56" @@ -1210,6 +1250,15 @@ version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "resolv-conf" version = "0.6.3" @@ -1226,6 +1275,18 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1253,9 +1314,9 @@ version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -1333,15 +1394,26 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.18", + "quote 1.0.6", + "unicode-xid 0.2.0", ] [[package]] @@ -1350,10 +1422,24 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", + "unicode-xid 0.2.0", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.8", ] [[package]] @@ -1364,9 +1450,9 @@ dependencies = [ "actix-web", "lazy_static", "maud", + "proptest", + "proptest-derive", "pulldown-cmark", - "quickcheck", - "quickcheck_macros", "rand", "serde", "serde_urlencoded", @@ -1390,9 +1476,9 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "893582086c2f98cde18f906265a65b5030a074b1046c674ae898be6519a7f479" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -1549,6 +1635,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.0" @@ -1572,6 +1664,15 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index d08aaee..b4ca8a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,5 +28,5 @@ features = ["actix-web"] [dev-dependencies] lazy_static = "~1.4" -quickcheck = "~0.9" -quickcheck_macros = "~0.9" +proptest = "~0.10" +proptest-derive = "~0.2" diff --git a/proptest-regressions/twelve.txt b/proptest-regressions/twelve.txt new file mode 100644 index 0000000..320b1b0 --- /dev/null +++ b/proptest-regressions/twelve.txt @@ -0,0 +1,9 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ee722004cd7dc5c8cfa8af5d83428a3c76f247525f738520e197ed67a0679b17 # shrinks to env_port = "", default_port = 0 +cc 8a8acaaebfe4e37af7655f14c3d9a394b2b3cc700936e36348158018ae8795ea # shrinks to env_port = "𐠁{�\"Tz`ﰃQ𑴉:*꠶䌁.", default_port = 42943 +cc f4d12f4f852ebcc99f6aa1a5208a392414d31dd98f025a7a905e1ef800d963af # shrinks to default_port = 65 diff --git a/src/twelve.rs b/src/twelve.rs index 273b17f..7803450 100644 --- a/src/twelve.rs +++ b/src/twelve.rs @@ -8,6 +8,8 @@ //! [1]: https://12factor.net/ //! [2]: https://12factor.net/config +#[cfg(test)] +use proptest_derive::Arbitrary; use std::env; use std::io; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; @@ -47,6 +49,7 @@ pub enum Error { /// Listening on this address will bind to both the ip4 and ip6 addresses on the /// current host, assuming both ip4 and ip6 are supported. #[derive(Debug, Clone)] +#[cfg_attr(test, derive(Arbitrary))] pub struct PortAddr { /// When used in an std::net::SocketAddr context, this is the port number to /// bind on. @@ -104,8 +107,7 @@ pub fn port(default_port: u16) -> Result { #[cfg(test)] mod tests { use lazy_static::lazy_static; - use quickcheck::{Arbitrary, Gen, TestResult}; - use quickcheck_macros::quickcheck; + use proptest::prelude::*; use std::env; use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -113,39 +115,33 @@ mod tests { use super::*; - impl Arbitrary for PortAddr { - fn arbitrary(g: &mut G) -> Self { - Self { - port: u16::arbitrary(g), - } - } - } - - #[quickcheck] - fn port_addr_as_socket_addr_has_v4(addr: PortAddr) -> bool { - let socket_addrs = addr.to_socket_addrs().unwrap().collect::>(); + proptest! { + #[test] + fn port_addr_as_socket_addr_has_v4(addr: PortAddr) { + let socket_addrs: Vec<_> = addr.to_socket_addrs().unwrap().collect(); - socket_addrs - .iter() - .any(|&socket_addr| socket_addr.is_ipv4()) - } + assert!(socket_addrs + .iter() + .any(|&socket_addr| socket_addr.is_ipv4())); + } - #[quickcheck] - fn port_addr_as_socket_addr_has_v6(addr: PortAddr) -> bool { - let socket_addrs = addr.to_socket_addrs().unwrap().collect::>(); + #[test] + fn port_addr_as_socket_addr_has_v6(addr: PortAddr) { + let socket_addrs: Vec<_> = addr.to_socket_addrs().unwrap().collect(); - socket_addrs - .iter() - .any(|&socket_addr| socket_addr.is_ipv6()) - } + assert!(socket_addrs + .iter() + .any(|&socket_addr| socket_addr.is_ipv6())); + } - #[quickcheck] - fn port_addr_as_socket_addr_all_have_port(addr: PortAddr) -> bool { - let socket_addrs = addr.to_socket_addrs().unwrap().collect::>(); + #[test] + fn port_addr_as_socket_addr_all_have_port(addr: PortAddr) { + let socket_addrs: Vec<_> = addr.to_socket_addrs().unwrap().collect(); - socket_addrs - .iter() - .all(|&socket_addr| socket_addr.port() == addr.port) + assert!(socket_addrs + .iter() + .all(|&socket_addr| socket_addr.port() == addr.port)); + } } #[derive(Default)] @@ -161,10 +157,10 @@ mod tests { lazy_static! { // The tests in this module manipulate a global, shared, external - // resource (the PORT environment variable). The quickcheck tool - // attempts to accelerate testing by running multiple threads, but this - // causes race conditions as test A stomps on state used by test B. - // Serialize tests through a mutex. + // resource (the PORT environment variable). The proptest tool attempts + // to accelerate testing by running multiple threads, but this causes + // race conditions as test A stomps on state used by test B. Serialize + // tests through a mutex. // // Huge hack. static ref ENV_MUTEX: Mutex = Mutex::new(Runner::default()); @@ -175,48 +171,47 @@ mod tests { ENV_MUTEX.lock().unwrap().run(f) } - #[quickcheck] - fn port_preserves_numeric_values(env_port: u16, default_port: u16) -> TestResult { - if env_port == default_port { - return TestResult::discard(); - } + proptest! { + #[test] + fn port_preserves_numeric_values(env_port: u16, default_port: u16) { + prop_assume!(env_port != default_port); - env_locked(|| { - env::set_var("PORT", env_port.to_string()); + env_locked(|| { + env::set_var("PORT", env_port.to_string()); - let read_port = port(default_port).unwrap(); + let read_port = port(default_port).unwrap(); - TestResult::from_bool(read_port.port == env_port) - }) - } - - #[quickcheck] - fn port_rejects_strings(env_port: String, default_port: u16) -> TestResult { - if env_port.contains("\x00") { - return TestResult::discard(); - } - if env_port.parse::().is_ok() { - return TestResult::discard(); + assert_eq!(read_port.port, env_port); + }); } - env_locked(|| { - env::set_var("PORT", env_port.to_string()); + #[test] + fn port_rejects_strings(env_port: String, default_port: u16) { + // Reject any sample with a NUL byte; env::set_var (well, libc) + // can't cope. + prop_assume!(!env_port.contains("\x00")); + // Reject any sample that _should_ parse cleanly. + prop_assume!(env_port.parse::().is_err()); - let port_result = port(default_port); + env_locked(|| { + env::set_var("PORT", env_port.to_string()); - TestResult::from_bool(port_result.is_err()) - }) - } + let port_result = port(default_port); - #[quickcheck] - fn port_uses_default(default_port: u16) -> bool { - env_locked(|| { - env::remove_var("PORT"); + assert!(port_result.is_err()); + }); + } - let read_port = port(default_port).unwrap(); + #[test] + fn port_uses_default(default_port: u16) { + env_locked(|| { + env::remove_var("PORT"); - read_port.port == default_port - }) + let read_port = port(default_port).unwrap(); + + assert_eq!(default_port, read_port.port); + }); + } } #[test] -- cgit v1.2.3