From cdccfb1a0e668d6e12a08cf4d7e58e971e8381f5 Mon Sep 17 00:00:00 2001 From: McKenna Date: Mon, 9 Oct 2023 11:53:44 -0400 Subject: [PATCH 1/6] cifuzz integration --- .github/workflows/cifuzz.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/cifuzz.yml diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml new file mode 100644 index 00000000..850541a0 --- /dev/null +++ b/.github/workflows/cifuzz.yml @@ -0,0 +1,36 @@ +name: CIFuzz +on: [pull_request] +permissions: {} +jobs: + Fuzzing: + runs-on: ubuntu-latest + permissions: + security-events: write + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'icalendar' + language: python + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'icalendar' + language: python + fuzz-seconds: 600 + output-sarif: true + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts + - name: Upload Sarif + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v2 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif + From 44ae07eb7d0520e4ac18d77131662a753afcbd48 Mon Sep 17 00:00:00 2001 From: McKenna Date: Mon, 9 Oct 2023 12:43:25 -0400 Subject: [PATCH 2/6] Migrated fuzzing harnesses and build script to fuzzing dir --- src/icalendar/fuzzing/build.sh | 25 ++++++ src/icalendar/fuzzing/enhanced_fdp.py | 111 ++++++++++++++++++++++++++ src/icalendar/fuzzing/ical_fuzzer.py | 46 +++++++++++ 3 files changed, 182 insertions(+) create mode 100755 src/icalendar/fuzzing/build.sh create mode 100644 src/icalendar/fuzzing/enhanced_fdp.py create mode 100644 src/icalendar/fuzzing/ical_fuzzer.py diff --git a/src/icalendar/fuzzing/build.sh b/src/icalendar/fuzzing/build.sh new file mode 100755 index 00000000..09c317ce --- /dev/null +++ b/src/icalendar/fuzzing/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash -eu +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +cd "$SRC"/icalendar +pip3 install . + +# Build fuzzers in $OUT +for fuzzer in $(find $SRC -name 'src/icalendar/fuzzing/*_fuzzer.py');do + compile_python_fuzzer "$fuzzer" +done +zip -q $OUT/ical_fuzzer_seed_corpus.zip $SRC/corpus/* diff --git a/src/icalendar/fuzzing/enhanced_fdp.py b/src/icalendar/fuzzing/enhanced_fdp.py new file mode 100644 index 00000000..3c5362be --- /dev/null +++ b/src/icalendar/fuzzing/enhanced_fdp.py @@ -0,0 +1,111 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +""" +Defines the EnhancedFuzzedDataProvider +""" +from contextlib import contextmanager +from enum import Enum +from io import BytesIO, StringIO +from tempfile import NamedTemporaryFile +from typing import Optional, Union + +from atheris import FuzzedDataProvider + + +class EnhancedFuzzedDataProvider(FuzzedDataProvider): + """ + Extends the functionality of FuzzedDataProvider + """ + + def _consume_random_count(self) -> int: + """ + :return: A count of bytes that is strictly in range 0<=n<=remaining_bytes + """ + return self.ConsumeIntInRange(0, self.remaining_bytes()) + + def _consume_file_data(self, all_data: bool, as_bytes: bool) -> Union[bytes, str]: + """ + Consumes data for a file + :param all_data: Whether to consume all remaining bytes from the buffer + :param as_bytes: Consumed output is bytes if true, otherwise a string + :return: The consumed output + """ + if all_data: + file_data = self.ConsumeRemainingBytes() if as_bytes else self.ConsumeRemainingString() + else: + file_data = self.ConsumeRandomBytes() if as_bytes else self.ConsumeRandomString() + + return file_data + + def ConsumeRandomBytes(self) -> bytes: + """ + Consume a 'random' count of the remaining bytes + :return: 0<=n<=remaining_bytes bytes + """ + return self.ConsumeBytes(self._consume_random_count()) + + def ConsumeRemainingBytes(self) -> bytes: + """ + :return: The remaining buffer + """ + return self.ConsumeBytes(self.remaining_bytes()) + + def ConsumeRandomString(self) -> str: + """ + Consume a 'random' length string, excluding surrogates + :return: The string + """ + return self.ConsumeUnicodeNoSurrogates(self._consume_random_count()) + + def ConsumeRemainingString(self) -> str: + """ + :return: The remaining buffer, as a string without surrogates + """ + return self.ConsumeUnicodeNoSurrogates(self.remaining_bytes()) + + def PickValueInEnum(self, enum): + return self.PickValueInList([e.value for e in enum]) + + @contextmanager + def ConsumeMemoryFile(self, all_data: bool, as_bytes: bool) -> Union[BytesIO, StringIO]: + """ + Consumes a file-like object, that resides entirely in memory + :param all_data: Whether to populate the file with all remaining data or not + :param as_bytes: Whether the file should hold bytes or strings + :return: The in-memory file + """ + file_data = self._consume_file_data(all_data, as_bytes) + file = BytesIO(file_data) if as_bytes else StringIO(file_data) + yield file + file.close() + + @contextmanager + def ConsumeTemporaryFile(self, all_data: bool, as_bytes: bool, suffix: Optional[str] = None) -> str: + """ + Consumes a temporary file, handling its deletion + :param all_data: Whether to populate the file with all remaining data or not + :param as_bytes: Whether the file should hold bytes or strings + :param suffix: A suffix to use for the generated file, e.g. 'txt' + :return: The path to the temporary file + """ + file_data = self._consume_file_data(all_data, as_bytes) + mode = 'w+b' if as_bytes else 'w+' + tfile = NamedTemporaryFile(mode=mode, suffix=suffix) + tfile.write(file_data) + tfile.seek(0) + tfile.flush() + yield tfile.name + tfile.close() diff --git a/src/icalendar/fuzzing/ical_fuzzer.py b/src/icalendar/fuzzing/ical_fuzzer.py new file mode 100644 index 00000000..bb6e2205 --- /dev/null +++ b/src/icalendar/fuzzing/ical_fuzzer.py @@ -0,0 +1,46 @@ +#!/usr/bin/python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +import atheris +import sys + +with atheris.instrument_imports(include=['icalendar']): + from icalendar import Calendar + +from enhanced_fdp import EnhancedFuzzedDataProvider + + +def TestOneInput(data): + fdp = EnhancedFuzzedDataProvider(data) + try: + Calendar.from_ical(fdp.ConsumeRemainingString()) + for event in Calendar.walk('VEVENT'): + event.to_ical().decode('utf-8') + except ValueError as e: + if "component" in str(e) or "parse" in str(e) or "Expected datetime" in str(e): + return -1 + raise e + except IndexError: + return -1 + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() From 16e6eb0d73b618d72599c13941621d2209b6ca50 Mon Sep 17 00:00:00 2001 From: McKenna Date: Mon, 9 Oct 2023 12:49:28 -0400 Subject: [PATCH 3/6] Capture more library-raised exceptions --- src/icalendar/fuzzing/ical_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icalendar/fuzzing/ical_fuzzer.py b/src/icalendar/fuzzing/ical_fuzzer.py index bb6e2205..6227f9ed 100644 --- a/src/icalendar/fuzzing/ical_fuzzer.py +++ b/src/icalendar/fuzzing/ical_fuzzer.py @@ -30,7 +30,7 @@ def TestOneInput(data): for event in Calendar.walk('VEVENT'): event.to_ical().decode('utf-8') except ValueError as e: - if "component" in str(e) or "parse" in str(e) or "Expected datetime" in str(e): + if "component" in str(e) or "parse" in str(e) or "Expected" in str(e): return -1 raise e except IndexError: From a260eac50ee2999b16417d1bdac28acdee1fd046 Mon Sep 17 00:00:00 2001 From: McKenna Date: Mon, 9 Oct 2023 14:21:14 -0400 Subject: [PATCH 4/6] Updated fuzzer for more coverage --- src/icalendar/fuzzing/enhanced_fdp.py | 111 -------------------------- src/icalendar/fuzzing/ical_fuzzer.py | 19 ++--- 2 files changed, 10 insertions(+), 120 deletions(-) delete mode 100644 src/icalendar/fuzzing/enhanced_fdp.py diff --git a/src/icalendar/fuzzing/enhanced_fdp.py b/src/icalendar/fuzzing/enhanced_fdp.py deleted file mode 100644 index 3c5362be..00000000 --- a/src/icalendar/fuzzing/enhanced_fdp.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -""" -Defines the EnhancedFuzzedDataProvider -""" -from contextlib import contextmanager -from enum import Enum -from io import BytesIO, StringIO -from tempfile import NamedTemporaryFile -from typing import Optional, Union - -from atheris import FuzzedDataProvider - - -class EnhancedFuzzedDataProvider(FuzzedDataProvider): - """ - Extends the functionality of FuzzedDataProvider - """ - - def _consume_random_count(self) -> int: - """ - :return: A count of bytes that is strictly in range 0<=n<=remaining_bytes - """ - return self.ConsumeIntInRange(0, self.remaining_bytes()) - - def _consume_file_data(self, all_data: bool, as_bytes: bool) -> Union[bytes, str]: - """ - Consumes data for a file - :param all_data: Whether to consume all remaining bytes from the buffer - :param as_bytes: Consumed output is bytes if true, otherwise a string - :return: The consumed output - """ - if all_data: - file_data = self.ConsumeRemainingBytes() if as_bytes else self.ConsumeRemainingString() - else: - file_data = self.ConsumeRandomBytes() if as_bytes else self.ConsumeRandomString() - - return file_data - - def ConsumeRandomBytes(self) -> bytes: - """ - Consume a 'random' count of the remaining bytes - :return: 0<=n<=remaining_bytes bytes - """ - return self.ConsumeBytes(self._consume_random_count()) - - def ConsumeRemainingBytes(self) -> bytes: - """ - :return: The remaining buffer - """ - return self.ConsumeBytes(self.remaining_bytes()) - - def ConsumeRandomString(self) -> str: - """ - Consume a 'random' length string, excluding surrogates - :return: The string - """ - return self.ConsumeUnicodeNoSurrogates(self._consume_random_count()) - - def ConsumeRemainingString(self) -> str: - """ - :return: The remaining buffer, as a string without surrogates - """ - return self.ConsumeUnicodeNoSurrogates(self.remaining_bytes()) - - def PickValueInEnum(self, enum): - return self.PickValueInList([e.value for e in enum]) - - @contextmanager - def ConsumeMemoryFile(self, all_data: bool, as_bytes: bool) -> Union[BytesIO, StringIO]: - """ - Consumes a file-like object, that resides entirely in memory - :param all_data: Whether to populate the file with all remaining data or not - :param as_bytes: Whether the file should hold bytes or strings - :return: The in-memory file - """ - file_data = self._consume_file_data(all_data, as_bytes) - file = BytesIO(file_data) if as_bytes else StringIO(file_data) - yield file - file.close() - - @contextmanager - def ConsumeTemporaryFile(self, all_data: bool, as_bytes: bool, suffix: Optional[str] = None) -> str: - """ - Consumes a temporary file, handling its deletion - :param all_data: Whether to populate the file with all remaining data or not - :param as_bytes: Whether the file should hold bytes or strings - :param suffix: A suffix to use for the generated file, e.g. 'txt' - :return: The path to the temporary file - """ - file_data = self._consume_file_data(all_data, as_bytes) - mode = 'w+b' if as_bytes else 'w+' - tfile = NamedTemporaryFile(mode=mode, suffix=suffix) - tfile.write(file_data) - tfile.seek(0) - tfile.flush() - yield tfile.name - tfile.close() diff --git a/src/icalendar/fuzzing/ical_fuzzer.py b/src/icalendar/fuzzing/ical_fuzzer.py index 6227f9ed..d524920a 100644 --- a/src/icalendar/fuzzing/ical_fuzzer.py +++ b/src/icalendar/fuzzing/ical_fuzzer.py @@ -20,22 +20,23 @@ with atheris.instrument_imports(include=['icalendar']): from icalendar import Calendar -from enhanced_fdp import EnhancedFuzzedDataProvider - def TestOneInput(data): - fdp = EnhancedFuzzedDataProvider(data) + fdp = atheris.FuzzedDataProvider(data) try: - Calendar.from_ical(fdp.ConsumeRemainingString()) - for event in Calendar.walk('VEVENT'): - event.to_ical().decode('utf-8') + b = fdp.ConsumeBool() + + cal = Calendar.from_ical(fdp.ConsumeString(fdp.remaining_bytes())) + + if b: + for event in cal.walk('VEVENT'): + event.to_ical().decode('utf-8') + else: + cal.to_ical() except ValueError as e: if "component" in str(e) or "parse" in str(e) or "Expected" in str(e): return -1 raise e - except IndexError: - return -1 - def main(): atheris.Setup(sys.argv, TestOneInput) From f5c83583002d10754a0a9af7699a68774602c018 Mon Sep 17 00:00:00 2001 From: McKenna Date: Thu, 19 Oct 2023 15:05:03 -0400 Subject: [PATCH 5/6] Omit fuzzing directory from doctest --- src/icalendar/tests/test_with_doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icalendar/tests/test_with_doctest.py b/src/icalendar/tests/test_with_doctest.py index 652198c3..d536ff9c 100644 --- a/src/icalendar/tests/test_with_doctest.py +++ b/src/icalendar/tests/test_with_doctest.py @@ -21,7 +21,7 @@ PYTHON_FILES = [ os.path.join(dirpath, filename) for dirpath, dirnames, filenames in os.walk(ICALENDAR_PATH) - for filename in filenames if filename.lower().endswith(".py") + for filename in filenames if filename.lower().endswith(".py") and 'fuzzing' not in dirpath ] MODULE_NAMES = [ From ed7ed2c561b2f47bb02adbf73c49a6dd79c4daac Mon Sep 17 00:00:00 2001 From: McKenna Date: Thu, 19 Oct 2023 15:15:47 -0400 Subject: [PATCH 6/6] Added to changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5d523cf1..b05c83f5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,7 +16,7 @@ Breaking changes: New features: -- ... +- Added fuzzing harnesses, for integration to OSSFuzz. Bug fixes: