diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 00000000..41deb44c --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,58 @@ +# Automatically generated by fuzz/generate-files.sh +name: Fuzz + +on: + push: + branches: + - master + - 'test-ci/**' + pull_request: + +jobs: + fuzz: + if: ${{ !github.event.act }} + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + fuzz_target: [ +simple_http, +minreq_http, + ] + steps: + - name: Install test dependencies + run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache-fuzz + with: + path: | + ~/.cargo/bin + fuzz/target + target + key: cache-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: 1.58 + override: true + profile: minimal + - name: fuzz + run: if [[ "${{ matrix.fuzz_target }}" =~ ^bitcoin ]]; then export RUSTFLAGS='--cfg=hashes_fuzz --cfg=secp256k1_fuzz'; fi + run: cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}" + - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }} + - uses: actions/upload-artifact@v2 + with: + name: executed_${{ matrix.fuzz_target }} + path: executed_${{ matrix.fuzz_target }} + + verify-execution: + if: ${{ !github.event.act }} + needs: fuzz + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + - name: Display structure of downloaded files + run: ls -R + - run: find executed_* -type f -exec cat {} + | sort > executed + - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index bbf5a84d..e162c065 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -1,20 +1,21 @@ [package] name = "jsonrpc-fuzz" +edition = "2018" version = "0.0.1" -authors = ["Automatically generated"] +authors = ["Generated by fuzz/generate-files.sh"] publish = false [package.metadata] cargo-fuzz = true -[features] -honggfuzz_fuzz = ["honggfuzz"] - [dependencies] -honggfuzz = { version = "0.5", optional = true, default-features = false } -jsonrpc = { path = ".." } +honggfuzz = { version = "0.5.55", default-features = false } +jsonrpc = { path = "..", features = ["minreq_http"] } [[bin]] name = "simple_http" path = "fuzz_targets/simple_http.rs" +[[bin]] +name = "minreq_http" +path = "fuzz_targets/minreq_http.rs" diff --git a/fuzz/cycle.sh b/fuzz/cycle.sh new file mode 100755 index 00000000..294f32b0 --- /dev/null +++ b/fuzz/cycle.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Continuosly cycle over fuzz targets running each for 1 hour. +# It uses chrt SCHED_IDLE so that other process takes priority. +# +# For hfuzz options see https://github.com/google/honggfuzz/blob/master/docs/USAGE.md + +set -e +REPO_DIR=$(git rev-parse --show-toplevel) +# shellcheck source=./fuzz-util.sh +source "$REPO_DIR/fuzz/fuzz-util.sh" + +while : +do + for targetFile in $(listTargetFiles); do + targetName=$(targetFileToName "$targetFile") + echo "Fuzzing target $targetName ($targetFile)" + + # fuzz for one hour + HFUZZ_RUN_ARGS='--run_time 3600' chrt -i 0 cargo hfuzz run "$targetName" + # minimize the corpus + HFUZZ_RUN_ARGS="-i hfuzz_workspace/$targetName/input/ -P -M" chrt -i 0 cargo hfuzz run "$targetName" + done +done diff --git a/fuzz/fuzz-util.sh b/fuzz/fuzz-util.sh new file mode 100755 index 00000000..804e0da9 --- /dev/null +++ b/fuzz/fuzz-util.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +REPO_DIR=$(git rev-parse --show-toplevel) + +listTargetFiles() { + pushd "$REPO_DIR/fuzz" > /dev/null || exit 1 + find fuzz_targets/ -type f -name "*.rs" + popd > /dev/null || exit 1 +} + +targetFileToName() { + echo "$1" \ + | sed 's/^fuzz_targets\///' \ + | sed 's/\.rs$//' \ + | sed 's/\//_/g' +} + +targetFileToHFuzzInputArg() { + baseName=$(basename "$1") + dirName="${baseName%.*}" + if [ -d "hfuzz_input/$dirName" ]; then + echo "HFUZZ_INPUT_ARGS=\"-f hfuzz_input/$FILE/input\"" + fi +} + +listTargetNames() { + for target in $(listTargetFiles); do + targetFileToName "$target" + done +} + +# Utility function to avoid CI failures on Windows +checkWindowsFiles() { + incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l) + if [ "$incorrectFilenames" -gt 0 ]; then + echo "Bailing early because there is a Windows-incompatible filename in the tree." + exit 2 + fi +} + +# Checks whether a fuzz case output some report, and dumps it in hex +checkReport() { + reportFile="hfuzz_workspace/$1/HONGGFUZZ.REPORT.TXT" + if [ -f "$reportFile" ]; then + cat "$reportFile" + for CASE in "hfuzz_workspace/$1/SIG"*; do + xxd -p -c10000 < "$CASE" + done + exit 1 + fi +} diff --git a/fuzz/fuzz.sh b/fuzz/fuzz.sh new file mode 100755 index 00000000..30e07fa2 --- /dev/null +++ b/fuzz/fuzz.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -ex + +REPO_DIR=$(git rev-parse --show-toplevel) + +# shellcheck source=./fuzz-util.sh +source "$REPO_DIR/fuzz/fuzz-util.sh" + +# Check that input files are correct Windows file names +checkWindowsFiles + +if [ "$1" == "" ]; then + targetFiles="$(listTargetFiles)" +else + targetFiles=fuzz_targets/"$1".rs +fi + +cargo --version +rustc --version + +# Testing +cargo install --force honggfuzz --no-default-features +for targetFile in $targetFiles; do + targetName=$(targetFileToName "$targetFile") + echo "Fuzzing target $targetName ($targetFile)" + if [ -d "hfuzz_input/$targetName" ]; then + HFUZZ_INPUT_ARGS="-f hfuzz_input/$targetName/input\"" + else + HFUZZ_INPUT_ARGS="" + fi + RUSTFLAGS="--cfg=jsonrpc_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run "$targetName" + + checkReport "$targetName" +done diff --git a/fuzz/fuzz_targets/minreq_http.rs b/fuzz/fuzz_targets/minreq_http.rs new file mode 100644 index 00000000..68eee3af --- /dev/null +++ b/fuzz/fuzz_targets/minreq_http.rs @@ -0,0 +1,62 @@ +extern crate jsonrpc; + +// Note, tests are empty if "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`. + +#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set. +fn do_test(data: &[u8]) { + #[cfg(jsonrpc_fuzz)] + { + use std::io; + + use jsonrpc::minreq_http::MinreqHttpTransport; + use jsonrpc::minreq_http::FUZZ_TCP_SOCK; + use jsonrpc::Client; + + *FUZZ_TCP_SOCK.lock().unwrap() = Some(io::Cursor::new(data.to_vec())); + + let t = MinreqHttpTransport::builder() + .url("localhost:123") + .expect("parse url") + .basic_auth("".to_string(), None) + .build(); + + let client = Client::with_transport(t); + let request = client.build_request("uptime", &[]); + let _ = client.send_request(request); + } +} + +fn main() { + loop { + honggfuzz::fuzz!(|data| { + do_test(data); + }); + } +} + +#[cfg(test)] +mod tests { + fn extend_vec_from_hex(hex: &str) -> Vec { + let mut out = vec![]; + let mut b = 0; + for (idx, c) in hex.as_bytes().iter().enumerate() { + b <<= 4; + match *c { + b'A'..=b'F' => b |= c - b'A' + 10, + b'a'..=b'f' => b |= c - b'a' + 10, + b'0'..=b'9' => b |= c - b'0', + _ => panic!("Bad hex"), + } + if (idx & 1) == 1 { + out.push(b); + b = 0; + } + } + out + } + + #[test] + fn duplicate_crash() { + super::do_test(&extend_vec_from_hex("00")); + } +} diff --git a/fuzz/fuzz_targets/simple_http.rs b/fuzz/fuzz_targets/simple_http.rs index 8b54bdbf..b27cc6a0 100644 --- a/fuzz/fuzz_targets/simple_http.rs +++ b/fuzz/fuzz_targets/simple_http.rs @@ -1,10 +1,10 @@ extern crate jsonrpc; -// Note, tests are empty "fuzzing" is not set but still show up in output of `cargo test --workspace`. +// Note, tests are if empty "jsonrpc_fuzz" is not set but still show up in output of `cargo test --workspace`. -#[allow(unused_variables)] // `data` is not used when "fuzzing" is not set. +#[allow(unused_variables)] // `data` is not used when "jsonrpc_fuzz" is not set. fn do_test(data: &[u8]) { - #[cfg(fuzzing)] + #[cfg(jsonrpc_fuzz)] { use std::io; @@ -26,7 +26,6 @@ fn do_test(data: &[u8]) { } } -#[cfg(feature = "honggfuzz")] fn main() { loop { honggfuzz::fuzz!(|data| { diff --git a/fuzz/generate-files.sh b/fuzz/generate-files.sh new file mode 100755 index 00000000..279002d6 --- /dev/null +++ b/fuzz/generate-files.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -e + +REPO_DIR=$(git rev-parse --show-toplevel) + +# shellcheck source=./fuzz-util.sh +source "$REPO_DIR/fuzz/fuzz-util.sh" + +# 1. Generate fuzz/Cargo.toml +cat > "$REPO_DIR/fuzz/Cargo.toml" <> "$REPO_DIR/fuzz/Cargo.toml" < "$REPO_DIR/.github/workflows/fuzz.yml" <executed_\${{ matrix.fuzz_target }} + - uses: actions/upload-artifact@v2 + with: + name: executed_\${{ matrix.fuzz_target }} + path: executed_\${{ matrix.fuzz_target }} + + verify-execution: + if: \${{ !github.event.act }} + needs: fuzz + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + - name: Display structure of downloaded files + run: ls -R + - run: find executed_* -type f -exec cat {} + | sort > executed + - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed +EOF diff --git a/fuzz/travis-fuzz.sh b/fuzz/travis-fuzz.sh deleted file mode 100755 index 71bff44a..00000000 --- a/fuzz/travis-fuzz.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -e - -# Check that input files are correct Windows file names -incorrectFilenames=$(find . -type f -name "*,*" -o -name "*:*" -o -name "*<*" -o -name "*>*" -o -name "*|*" -o -name "*\?*" -o -name "*\**" -o -name "*\"*" | wc -l) - -if [ ${incorrectFilenames} -gt 0 ]; then - echo 'Exiting early due to Windows-incompatible filenames in this tree. If this is happening' - echo 'to you on a local run, deleting the `hfuzz_workspace/` directory will probably fix it.' - exit 2 -fi - -if [ "$1" == "" ]; then - TARGETS=fuzz_targets/* -else - TARGETS=fuzz_targets/"$1".rs -fi - -cargo --version -rustc --version - -# Testing -cargo install --force honggfuzz --no-default-features -for TARGET in $TARGETS; do - echo "Fuzzing target $TARGET" - FILENAME=$(basename $TARGET) - FILE="${FILENAME%.*}" - if [ -d hfuzz_input/$FILE ]; then - HFUZZ_INPUT_ARGS="-f hfuzz_input/$FILE/input" - fi - HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE - - if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then - cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT - for CASE in hfuzz_workspace/$FILE/SIG*; do - cat $CASE | xxd -p -c1000 - done - exit 1 - fi -done diff --git a/src/http/minreq_http.rs b/src/http/minreq_http.rs index da46fea8..2070e0cd 100644 --- a/src/http/minreq_http.rs +++ b/src/http/minreq_http.rs @@ -3,6 +3,10 @@ //! //! [minreq]: +#[cfg(jsonrpc_fuzz)] +use std::io::{self, Read, Write}; +#[cfg(jsonrpc_fuzz)] +use std::sync::Mutex; use std::time::Duration; use std::{error, fmt}; @@ -11,7 +15,10 @@ use crate::{Request, Response}; const DEFAULT_URL: &str = "http://localhost"; const DEFAULT_PORT: u16 = 8332; // the default RPC port for bitcoind. +#[cfg(not(jsonrpc_fuzz))] const DEFAULT_TIMEOUT_SECONDS: u64 = 15; +#[cfg(jsonrpc_fuzz)] +const DEFAULT_TIMEOUT_SECONDS: u64 = 1; /// An HTTP transport that uses [`minreq`] and is useful for running a bitcoind RPC client. #[derive(Clone, Debug)] @@ -199,6 +206,36 @@ impl From for crate::Error { } } +/// Global mutex used by the fuzzing harness to inject data into the read end of the TCP stream. +#[cfg(jsonrpc_fuzz)] +pub static FUZZ_TCP_SOCK: Mutex>>> = Mutex::new(None); + +#[cfg(jsonrpc_fuzz)] +#[derive(Clone, Debug)] +struct TcpStream; + +#[cfg(jsonrpc_fuzz)] +mod impls { + use super::*; + + impl Read for TcpStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match *FUZZ_TCP_SOCK.lock().unwrap() { + Some(ref mut cursor) => io::Read::read(cursor, buf), + None => Ok(0), + } + } + } + impl Write for TcpStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + io::sink().write(buf) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/http/simple_http.rs b/src/http/simple_http.rs index c338cceb..3e2a1728 100644 --- a/src/http/simple_http.rs +++ b/src/http/simple_http.rs @@ -7,7 +7,7 @@ #[cfg(feature = "proxy")] use socks::Socks5Stream; use std::io::{BufRead, BufReader, Read, Write}; -#[cfg(not(fuzzing))] +#[cfg(not(jsonrpc_fuzz))] use std::net::TcpStream; use std::net::{SocketAddr, ToSocketAddrs}; use std::sync::{Arc, Mutex, MutexGuard}; @@ -23,10 +23,10 @@ use crate::{Request, Response}; /// Absolute maximum content length allowed before cutting off the response. const FINAL_RESP_ALLOC: u64 = 1024 * 1024 * 1024; -#[cfg(not(fuzzing))] +#[cfg(not(jsonrpc_fuzz))] const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15); -#[cfg(fuzzing)] +#[cfg(jsonrpc_fuzz)] const DEFAULT_TIMEOUT: Duration = Duration::from_millis(1); /// Simple HTTP transport that implements the necessary subset of HTTP for @@ -620,14 +620,14 @@ impl From for crate::Error { } /// Global mutex used by the fuzzing harness to inject data into the read end of the TCP stream. -#[cfg(fuzzing)] +#[cfg(jsonrpc_fuzz)] pub static FUZZ_TCP_SOCK: Mutex>>> = Mutex::new(None); -#[cfg(fuzzing)] +#[cfg(jsonrpc_fuzz)] #[derive(Clone, Debug)] struct TcpStream; -#[cfg(fuzzing)] +#[cfg(jsonrpc_fuzz)] mod impls { use super::*; impl Read for TcpStream { @@ -662,17 +662,8 @@ mod impls { #[cfg(test)] mod tests { - #[cfg(not(feature = "proxy"))] - use serde_json::{Number, Value}; use std::net; - #[cfg(not(feature = "proxy"))] - use std::net::{Shutdown, TcpListener}; - #[cfg(feature = "proxy")] use std::str::FromStr; - #[cfg(not(feature = "proxy"))] - use std::sync::mpsc; - #[cfg(not(feature = "proxy"))] - use std::thread; use super::*; use crate::Client; @@ -776,9 +767,14 @@ mod tests { /// Test that the client will detect that a socket is closed and open a fresh one before sending /// the request - #[cfg(all(not(feature = "proxy"), not(fuzzing)))] + #[cfg(all(not(feature = "proxy"), not(jsonrpc_fuzz)))] #[test] fn request_to_closed_socket() { + use serde_json::{Number, Value}; + use std::net::{Shutdown, TcpListener}; + use std::sync::mpsc; + use std::thread; + let (tx, rx) = mpsc::sync_channel(1); thread::spawn(move || {