2020-03-07 05:09:10 +00:00
|
|
|
#!/bin/bash -eux
|
|
|
|
|
# Copyright 2020 Google Inc.
|
|
|
|
|
#
|
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
|
# limitations under the License.
|
|
|
|
|
#
|
|
|
|
|
################################################################################
|
|
|
|
|
|
|
|
|
|
# Note: This project creates Rust fuzz targets exclusively
|
|
|
|
|
|
[libra] Fix building failure (#3566)
After many days banging my head on FFI issues in rust,
I hereby present a fix to the issue.
Note that I've got some help, and I'm not sure I understand everything here.
But this is my understanding of what was not working, and how we fixed it.
The **problem** is that on Ubuntu 16 with llvm/clang 10,
we were **statically linking libc++** in [rocksdb][1]:
```rust
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
// ...
// remove lib prefix and .a postfix.
println!(
"cargo:rustc-link-lib=static={}",
&stdlib[3..stdlib.len() - 2]
);
```
This means that during building, when we reach building of rocksdb,
we import a number of symbols from libc++ (like [__muloti4][2])
that end up in the associated `.rlib` (rust obj file).
These symbols interestingly do not exist in libstdc++ which is used by gcc.
This is important because on linux (unlike mac), the rust toolchain is compiled with gcc.
So these intrinsics are not present in the linux rust toolchain,
and have been redeclared in the [compiler-builtins][3] crate.
So here is the problem:
* rust toolchain's defines these intrinsics functions
* libc++ defines these intrinsics functions
And the recipe for disaster:
* libc++ is statically linked in rocksdb, which means all the symbols are imported
* symbols in rocksdb's produced `.rlib` are colliding with the symbols from the rust toolchain `.rlib`
To fix this. Maybe we could have compiled the stuff with libstdc++?
But instead we:
1. removed the static linking
2. we linked libc++ at the very last moment via:
```rust
RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
```
At final linking time, the linker sees that the intrinsics are already defined in one of the `.rlib`
(produced by compiler-builtins) and so does not import these functions from libc++.
Actually, at this point it only statically link the functions that need to be linked.
It seems to work.
[1]: https://github.com/tikv/rust-rocksdb/blob/c79d2c2ac68f9ec40fa03faac3882baac38db314/librocksdb_sys/build.rs#L115
[2]: https://github.com/llvm-mirror/libcxx/blob/master/src/filesystem/int128_builtins.cpp
[3]: https://github.com/rust-lang/compiler-builtins/blob/e578d47247470935ebcc2bbfc81430b94f31ffd8/src/int/mul.rs#L107
2020-04-02 21:28:09 +00:00
|
|
|
# recipe:
|
|
|
|
|
# -------
|
|
|
|
|
# 1. we list all the fuzzers and save to a file fuzzer_list
|
|
|
|
|
# 2. we build the corpus for each fuzzer
|
|
|
|
|
# 3. we build all the fuzzers
|
|
|
|
|
|
|
|
|
|
# reset flags of OSS-Fuzz
|
2020-03-07 05:09:10 +00:00
|
|
|
export CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
|
|
|
|
|
export CXXFLAGS_EXTRA="-stdlib=libc++"
|
|
|
|
|
export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA"
|
|
|
|
|
|
[libra] Fix building failure (#3566)
After many days banging my head on FFI issues in rust,
I hereby present a fix to the issue.
Note that I've got some help, and I'm not sure I understand everything here.
But this is my understanding of what was not working, and how we fixed it.
The **problem** is that on Ubuntu 16 with llvm/clang 10,
we were **statically linking libc++** in [rocksdb][1]:
```rust
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
// ...
// remove lib prefix and .a postfix.
println!(
"cargo:rustc-link-lib=static={}",
&stdlib[3..stdlib.len() - 2]
);
```
This means that during building, when we reach building of rocksdb,
we import a number of symbols from libc++ (like [__muloti4][2])
that end up in the associated `.rlib` (rust obj file).
These symbols interestingly do not exist in libstdc++ which is used by gcc.
This is important because on linux (unlike mac), the rust toolchain is compiled with gcc.
So these intrinsics are not present in the linux rust toolchain,
and have been redeclared in the [compiler-builtins][3] crate.
So here is the problem:
* rust toolchain's defines these intrinsics functions
* libc++ defines these intrinsics functions
And the recipe for disaster:
* libc++ is statically linked in rocksdb, which means all the symbols are imported
* symbols in rocksdb's produced `.rlib` are colliding with the symbols from the rust toolchain `.rlib`
To fix this. Maybe we could have compiled the stuff with libstdc++?
But instead we:
1. removed the static linking
2. we linked libc++ at the very last moment via:
```rust
RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
```
At final linking time, the linker sees that the intrinsics are already defined in one of the `.rlib`
(produced by compiler-builtins) and so does not import these functions from libc++.
Actually, at this point it only statically link the functions that need to be linked.
It seems to work.
[1]: https://github.com/tikv/rust-rocksdb/blob/c79d2c2ac68f9ec40fa03faac3882baac38db314/librocksdb_sys/build.rs#L115
[2]: https://github.com/llvm-mirror/libcxx/blob/master/src/filesystem/int128_builtins.cpp
[3]: https://github.com/rust-lang/compiler-builtins/blob/e578d47247470935ebcc2bbfc81430b94f31ffd8/src/int/mul.rs#L107
2020-04-02 21:28:09 +00:00
|
|
|
# correct workdir
|
2020-03-07 05:09:10 +00:00
|
|
|
cd $SRC/libra/testsuite/libra-fuzzer
|
|
|
|
|
|
[libra] Fix building failure (#3566)
After many days banging my head on FFI issues in rust,
I hereby present a fix to the issue.
Note that I've got some help, and I'm not sure I understand everything here.
But this is my understanding of what was not working, and how we fixed it.
The **problem** is that on Ubuntu 16 with llvm/clang 10,
we were **statically linking libc++** in [rocksdb][1]:
```rust
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
// ...
// remove lib prefix and .a postfix.
println!(
"cargo:rustc-link-lib=static={}",
&stdlib[3..stdlib.len() - 2]
);
```
This means that during building, when we reach building of rocksdb,
we import a number of symbols from libc++ (like [__muloti4][2])
that end up in the associated `.rlib` (rust obj file).
These symbols interestingly do not exist in libstdc++ which is used by gcc.
This is important because on linux (unlike mac), the rust toolchain is compiled with gcc.
So these intrinsics are not present in the linux rust toolchain,
and have been redeclared in the [compiler-builtins][3] crate.
So here is the problem:
* rust toolchain's defines these intrinsics functions
* libc++ defines these intrinsics functions
And the recipe for disaster:
* libc++ is statically linked in rocksdb, which means all the symbols are imported
* symbols in rocksdb's produced `.rlib` are colliding with the symbols from the rust toolchain `.rlib`
To fix this. Maybe we could have compiled the stuff with libstdc++?
But instead we:
1. removed the static linking
2. we linked libc++ at the very last moment via:
```rust
RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
```
At final linking time, the linker sees that the intrinsics are already defined in one of the `.rlib`
(produced by compiler-builtins) and so does not import these functions from libc++.
Actually, at this point it only statically link the functions that need to be linked.
It seems to work.
[1]: https://github.com/tikv/rust-rocksdb/blob/c79d2c2ac68f9ec40fa03faac3882baac38db314/librocksdb_sys/build.rs#L115
[2]: https://github.com/llvm-mirror/libcxx/blob/master/src/filesystem/int128_builtins.cpp
[3]: https://github.com/rust-lang/compiler-builtins/blob/e578d47247470935ebcc2bbfc81430b94f31ffd8/src/int/mul.rs#L107
2020-04-02 21:28:09 +00:00
|
|
|
# fetch all dependencies (needed for patching rocksdb)
|
|
|
|
|
cargo fetch
|
|
|
|
|
|
|
|
|
|
# patch rocksdb to not link libc++ statically
|
|
|
|
|
sed -i "s/link_cpp(&mut build)/build.cpp_link_stdlib(None)/" \
|
|
|
|
|
/rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/72e45c3/librocksdb_sys/build.rs
|
|
|
|
|
# so now we need to link libc++ at the end
|
|
|
|
|
export RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
|
|
|
|
|
|
|
|
|
|
# 1. list fuzzers
|
2020-03-07 05:09:10 +00:00
|
|
|
cargo run --bin libra-fuzzer list --no-desc > fuzzer_list
|
|
|
|
|
|
[libra] Fix building failure (#3566)
After many days banging my head on FFI issues in rust,
I hereby present a fix to the issue.
Note that I've got some help, and I'm not sure I understand everything here.
But this is my understanding of what was not working, and how we fixed it.
The **problem** is that on Ubuntu 16 with llvm/clang 10,
we were **statically linking libc++** in [rocksdb][1]:
```rust
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
// ...
// remove lib prefix and .a postfix.
println!(
"cargo:rustc-link-lib=static={}",
&stdlib[3..stdlib.len() - 2]
);
```
This means that during building, when we reach building of rocksdb,
we import a number of symbols from libc++ (like [__muloti4][2])
that end up in the associated `.rlib` (rust obj file).
These symbols interestingly do not exist in libstdc++ which is used by gcc.
This is important because on linux (unlike mac), the rust toolchain is compiled with gcc.
So these intrinsics are not present in the linux rust toolchain,
and have been redeclared in the [compiler-builtins][3] crate.
So here is the problem:
* rust toolchain's defines these intrinsics functions
* libc++ defines these intrinsics functions
And the recipe for disaster:
* libc++ is statically linked in rocksdb, which means all the symbols are imported
* symbols in rocksdb's produced `.rlib` are colliding with the symbols from the rust toolchain `.rlib`
To fix this. Maybe we could have compiled the stuff with libstdc++?
But instead we:
1. removed the static linking
2. we linked libc++ at the very last moment via:
```rust
RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
```
At final linking time, the linker sees that the intrinsics are already defined in one of the `.rlib`
(produced by compiler-builtins) and so does not import these functions from libc++.
Actually, at this point it only statically link the functions that need to be linked.
It seems to work.
[1]: https://github.com/tikv/rust-rocksdb/blob/c79d2c2ac68f9ec40fa03faac3882baac38db314/librocksdb_sys/build.rs#L115
[2]: https://github.com/llvm-mirror/libcxx/blob/master/src/filesystem/int128_builtins.cpp
[3]: https://github.com/rust-lang/compiler-builtins/blob/e578d47247470935ebcc2bbfc81430b94f31ffd8/src/int/mul.rs#L107
2020-04-02 21:28:09 +00:00
|
|
|
# 2. build corpus and move to $OUT
|
2020-03-07 05:09:10 +00:00
|
|
|
cat fuzzer_list | while read -r line
|
|
|
|
|
do
|
|
|
|
|
cargo run --bin libra-fuzzer generate -n 128 $line
|
|
|
|
|
zip -r $OUT/"$line"_seed_corpus.zip fuzz/corpus/$line
|
|
|
|
|
rm -r fuzz/corpus/$line
|
|
|
|
|
done
|
|
|
|
|
|
[libra] Fix building failure (#3566)
After many days banging my head on FFI issues in rust,
I hereby present a fix to the issue.
Note that I've got some help, and I'm not sure I understand everything here.
But this is my understanding of what was not working, and how we fixed it.
The **problem** is that on Ubuntu 16 with llvm/clang 10,
we were **statically linking libc++** in [rocksdb][1]:
```rust
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
// ...
// remove lib prefix and .a postfix.
println!(
"cargo:rustc-link-lib=static={}",
&stdlib[3..stdlib.len() - 2]
);
```
This means that during building, when we reach building of rocksdb,
we import a number of symbols from libc++ (like [__muloti4][2])
that end up in the associated `.rlib` (rust obj file).
These symbols interestingly do not exist in libstdc++ which is used by gcc.
This is important because on linux (unlike mac), the rust toolchain is compiled with gcc.
So these intrinsics are not present in the linux rust toolchain,
and have been redeclared in the [compiler-builtins][3] crate.
So here is the problem:
* rust toolchain's defines these intrinsics functions
* libc++ defines these intrinsics functions
And the recipe for disaster:
* libc++ is statically linked in rocksdb, which means all the symbols are imported
* symbols in rocksdb's produced `.rlib` are colliding with the symbols from the rust toolchain `.rlib`
To fix this. Maybe we could have compiled the stuff with libstdc++?
But instead we:
1. removed the static linking
2. we linked libc++ at the very last moment via:
```rust
RUSTFLAGS="-C link-arg=-L/usr/local/lib -C link-arg=-lc++"
```
At final linking time, the linker sees that the intrinsics are already defined in one of the `.rlib`
(produced by compiler-builtins) and so does not import these functions from libc++.
Actually, at this point it only statically link the functions that need to be linked.
It seems to work.
[1]: https://github.com/tikv/rust-rocksdb/blob/c79d2c2ac68f9ec40fa03faac3882baac38db314/librocksdb_sys/build.rs#L115
[2]: https://github.com/llvm-mirror/libcxx/blob/master/src/filesystem/int128_builtins.cpp
[3]: https://github.com/rust-lang/compiler-builtins/blob/e578d47247470935ebcc2bbfc81430b94f31ffd8/src/int/mul.rs#L107
2020-04-02 21:28:09 +00:00
|
|
|
# rust libfuzzer flags (https://github.com/rust-fuzz/libfuzzer/blob/master/build.rs#L12)
|
|
|
|
|
export CUSTOM_LIBFUZZER_PATH="$LIB_FUZZING_ENGINE_DEPRECATED"
|
|
|
|
|
export CUSTOM_LIBFUZZER_STD_CXX=c++
|
|
|
|
|
# export CUSTOM_LIBFUZZER_STD_CXX=none
|
|
|
|
|
|
|
|
|
|
# export fuzzing flags
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS --cfg fuzzing" # used to change code logic
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cdebug-assertions" # to get debug_assert in rust
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Zsanitizer=address" # address sanitizer (ASAN)
|
|
|
|
|
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cdebuginfo=1"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cforce-frame-pointers"
|
|
|
|
|
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cpasses=sancov"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-level=4"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-trace-compares"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-inline-8bit-counters"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-trace-geps"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-prune-blocks=0"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-pc-table"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Clink-dead-code"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Cllvm-args=-sanitizer-coverage-stack-depth"
|
|
|
|
|
RUSTFLAGS="$RUSTFLAGS -Ccodegen-units=1"
|
|
|
|
|
|
|
|
|
|
export RUSTFLAGS
|
|
|
|
|
|
|
|
|
|
# 3. build all the fuzzers!
|
2020-03-07 05:09:10 +00:00
|
|
|
cat fuzzer_list | while read -r line
|
|
|
|
|
do
|
|
|
|
|
# build
|
|
|
|
|
export SINGLE_FUZZ_TARGET="$line"
|
2020-03-16 21:21:45 +00:00
|
|
|
cargo build --manifest-path fuzz/Cargo.toml --bin fuzzer_builder --release --target x86_64-unknown-linux-gnu
|
2020-03-07 05:09:10 +00:00
|
|
|
# move fuzzer to $OUT
|
2020-03-16 21:21:45 +00:00
|
|
|
mv $SRC/libra/target/x86_64-unknown-linux-gnu/release/fuzzer_builder $OUT/$SINGLE_FUZZ_TARGET
|
2020-03-07 05:09:10 +00:00
|
|
|
done
|