@@ -9,23 +9,20 @@ set -euo pipefail |
9 | 9 | # Prerequisites # |
10 | 10 | ################# |
11 | 11 |
|
12 | | -for cmd in python3 git wget rsync; do |
| 12 | +for cmd in python3 git wget zip; do |
13 | 13 | command -v "$cmd" >/dev/null 2>&1 || { |
14 | 14 | printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2 |
15 | 15 | exit 1 |
16 | 16 | } |
17 | 17 | done |
18 | 18 |
|
19 | | -SEED_DATA_DIR="$SRC/seed_data" |
20 | | -mkdir -p "$SEED_DATA_DIR" |
21 | | - |
22 | 19 | ############# |
23 | 20 | # Functions # |
24 | 21 | ############# |
25 | 22 |
|
26 | 23 | download_and_concatenate_common_dictionaries() { |
27 | 24 | # Assign the first argument as the target file where all contents will be concatenated |
28 | | - target_file="$1" |
| 25 | + local target_file="$1" |
29 | 26 |
|
30 | 27 | # Shift the arguments so the first argument (target_file path) is removed |
31 | 28 | # and only URLs are left for the loop below. |
@@ -38,22 +35,61 @@ download_and_concatenate_common_dictionaries() { |
38 | 35 | done |
39 | 36 | } |
40 | 37 |
|
41 | | -fetch_seed_corpora() { |
42 | | - # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. |
43 | | - git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && |
44 | | - rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && |
45 | | - rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible. |
| 38 | +create_seed_corpora_zips() { |
| 39 | + local seed_corpora_dir="$1" |
| 40 | + local output_zip |
| 41 | + for dir in "$seed_corpora_dir"/*; do |
| 42 | + if [ -d "$dir" ] && [ -n "$dir" ]; then |
| 43 | + output_zip="$SRC/$(basename "$dir")_seed_corpus.zip" |
| 44 | + printf '[%s] Zipping the contents of %s into %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dir" "$output_zip" |
| 45 | + zip -jur "$output_zip" "$dir"/* |
| 46 | + fi |
| 47 | + done |
| 48 | +} |
| 49 | + |
| 50 | +prepare_dictionaries_for_fuzz_targets() { |
| 51 | + local dictionaries_dir="$1" |
| 52 | + local fuzz_targets_dir="$2" |
| 53 | + local common_base_dictionary_filename="$WORK/__base.dict" |
| 54 | + |
| 55 | + printf '[%s] Copying .dict files from %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dictionaries_dir" "$SRC/" |
| 56 | + cp -v "$dictionaries_dir"/*.dict "$SRC/" |
| 57 | + |
| 58 | + download_and_concatenate_common_dictionaries "$common_base_dictionary_filename" \ |
| 59 | + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ |
| 60 | + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" |
| 61 | + |
| 62 | + find "$fuzz_targets_dir" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do |
| 63 | + if [[ -r "$common_base_dictionary_filename" ]]; then |
| 64 | + # Strip the `.py` extension from the filename and replace it with `.dict`. |
| 65 | + fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" |
| 66 | + local output_file="$SRC/$fuzz_harness_dictionary_filename" |
| 67 | + |
| 68 | + printf '[%s] Appending %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$common_base_dictionary_filename" "$output_file" |
| 69 | + if [[ -s "$output_file" ]]; then |
| 70 | + # If a dictionary file for this fuzzer already exists and is not empty, |
| 71 | + # we append a new line to the end of it before appending any new entries. |
| 72 | + # |
| 73 | + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error |
| 74 | + # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) |
| 75 | + # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 |
| 76 | + echo >>"$output_file" |
| 77 | + fi |
| 78 | + cat "$common_base_dictionary_filename" >>"$output_file" |
| 79 | + fi |
| 80 | + done |
46 | 81 | } |
47 | 82 |
|
48 | 83 | ######################## |
49 | 84 | # Main execution logic # |
50 | 85 | ######################## |
| 86 | +# Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. |
| 87 | +# We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. |
| 88 | +git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" |
51 | 89 |
|
52 | | -fetch_seed_corpora |
| 90 | +create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" |
53 | 91 |
|
54 | | -download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ |
55 | | - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ |
56 | | - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" |
| 92 | +prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" |
57 | 93 |
|
58 | 94 | # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. |
59 | 95 | python3 -m pip install --upgrade pip |
|