diff --git a/.github/actions/spelling/allow/allow.txt b/.github/actions/spelling/allow/allow.txt index bdbcfc923bd..442651d7c77 100644 --- a/.github/actions/spelling/allow/allow.txt +++ b/.github/actions/spelling/allow/allow.txt @@ -3,6 +3,7 @@ apc Apc bsd calt +CMMI ccmp changelog clickable @@ -37,6 +38,7 @@ img inlined It'd kje +libfuzzer liga lje Llast @@ -48,11 +50,13 @@ maxed mkmk mnt mru +noreply nje noreply ogonek ok'd overlined +pipeline postmodern ptys qof diff --git a/.github/actions/spelling/allow/apis.txt b/.github/actions/spelling/allow/apis.txt index 062bba9f622..192c3350fa2 100644 --- a/.github/actions/spelling/allow/apis.txt +++ b/.github/actions/spelling/allow/apis.txt @@ -30,6 +30,7 @@ DERR dlldata DONTADDTORECENT DWORDLONG +endfor enumset environstrings EXPCMDFLAGS @@ -120,6 +121,7 @@ oaidl ocidl ODR offsetof +onefuzz osver OSVERSIONINFOEXW otms diff --git a/.github/actions/spelling/allow/microsoft.txt b/.github/actions/spelling/allow/microsoft.txt index a961314685e..13197f167b2 100644 --- a/.github/actions/spelling/allow/microsoft.txt +++ b/.github/actions/spelling/allow/microsoft.txt @@ -39,6 +39,7 @@ MSVC muxc netcore osgvsowi +Onefuzz PFILETIME pgc pgo diff --git a/.github/actions/spelling/allow/names.txt b/.github/actions/spelling/allow/names.txt index 2a13d67badd..4965a52939b 100644 --- a/.github/actions/spelling/allow/names.txt +++ b/.github/actions/spelling/allow/names.txt @@ -8,6 +8,7 @@ dhowett Diviness dsafa duhowett +DXP ekg eryksun ethanschoonover @@ -69,6 +70,7 @@ sonpham stakx thereses Walisch +WDX Wellons Wirt Wojciech diff --git a/.github/actions/spelling/expect/web.txt b/.github/actions/spelling/expect/web.txt index 4b95ef2e627..60ae118cb48 100644 --- a/.github/actions/spelling/expect/web.txt +++ b/.github/actions/spelling/expect/web.txt @@ -17,6 +17,7 @@ mdtauk cppreference gfycat Guake +azurewebsites askubuntu dostips viewtopic diff --git a/.vsconfig b/.vsconfig index 79c5279a774..46aed406c83 100644 --- a/.vsconfig +++ b/.vsconfig @@ -25,6 +25,7 @@ "Microsoft.VisualStudio.Component.VC.Redist.14.Latest", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.ASAN", "Microsoft.VisualStudio.Component.VC.v142.x86.x64", "Microsoft.VisualStudio.Component.VC.v142.ARM64", "Microsoft.VisualStudio.ComponentGroup.UWP.VC", diff --git a/OpenConsole.sln b/OpenConsole.sln index 510b7a1a023..45613781537 100644 --- a/OpenConsole.sln +++ b/OpenConsole.sln @@ -1200,11 +1200,11 @@ Global {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Debug|x86.Build.0 = Debug|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|Any CPU.ActiveCfg = Debug|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|ARM.ActiveCfg = Debug|Win32 - {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|ARM64.ActiveCfg = Fuzzing|ARM64 - {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|DotNet_x64Test.ActiveCfg = Fuzzing|Win32 + {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|ARM64.ActiveCfg = Debug|ARM64 + {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|DotNet_x64Test.ActiveCfg = Debug|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|DotNet_x86Test.ActiveCfg = Debug|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|x64.ActiveCfg = Debug|x64 - {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|x86.ActiveCfg = Fuzzing|Win32 + {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Fuzzing|x86.ActiveCfg = Debug|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Release|Any CPU.ActiveCfg = Release|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Release|ARM.ActiveCfg = Release|Win32 {099193A0-1E43-4BBC-BA7F-7B351E1342DF}.Release|ARM64.ActiveCfg = Release|ARM64 @@ -1271,7 +1271,6 @@ Global {919544AC-D39B-463F-8414-3C3C67CF727C}.Fuzzing|DotNet_x64Test.ActiveCfg = Fuzzing|Win32 {919544AC-D39B-463F-8414-3C3C67CF727C}.Fuzzing|DotNet_x86Test.ActiveCfg = Fuzzing|Win32 {919544AC-D39B-463F-8414-3C3C67CF727C}.Fuzzing|x64.ActiveCfg = Fuzzing|x64 - {919544AC-D39B-463F-8414-3C3C67CF727C}.Fuzzing|x64.Build.0 = Fuzzing|x64 {919544AC-D39B-463F-8414-3C3C67CF727C}.Fuzzing|x86.ActiveCfg = Fuzzing|Win32 {919544AC-D39B-463F-8414-3C3C67CF727C}.Release|Any CPU.ActiveCfg = Release|Win32 {919544AC-D39B-463F-8414-3C3C67CF727C}.Release|ARM.ActiveCfg = Release|Win32 @@ -3241,13 +3240,10 @@ Global {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|Any CPU.ActiveCfg = Fuzzing|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|ARM.ActiveCfg = Fuzzing|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|ARM64.ActiveCfg = Fuzzing|ARM64 - {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|ARM64.Build.0 = Fuzzing|ARM64 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|DotNet_x64Test.ActiveCfg = Fuzzing|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|DotNet_x86Test.ActiveCfg = Fuzzing|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|x64.ActiveCfg = Fuzzing|x64 - {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|x64.Build.0 = Fuzzing|x64 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|x86.ActiveCfg = Fuzzing|Win32 - {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Fuzzing|x86.Build.0 = Fuzzing|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|Any CPU.ActiveCfg = Release|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|ARM.ActiveCfg = Release|Win32 {C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|ARM64.ActiveCfg = Release|ARM64 @@ -3321,7 +3317,6 @@ Global {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|DotNet_x64Test.ActiveCfg = Fuzzing|Win32 {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|DotNet_x86Test.ActiveCfg = Fuzzing|Win32 {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|x64.ActiveCfg = Fuzzing|x64 - {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|x64.Build.0 = Fuzzing|x64 {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|x86.ActiveCfg = Fuzzing|Win32 {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Fuzzing|x86.Build.0 = Fuzzing|Win32 {9CF74355-F018-4C19-81AD-9DC6B7F2C6F5}.Release|Any CPU.ActiveCfg = Release|Win32 diff --git a/build/Fuzz/notifications-ado.json b/build/Fuzz/notifications-ado.json new file mode 100644 index 00000000000..dc1a7218cc7 --- /dev/null +++ b/build/Fuzz/notifications-ado.json @@ -0,0 +1,34 @@ +{ + "config": { + "base_url": "https://dev.azure.com/microsoft/os", + "auth_token": "INSERT_PAT_HERE", + "project": "OpenConsole", + "type": "Bug", + "unique_fields": [ + "Microsoft.VSTS.Common.CustomString03" + ], + "comment": "This input caused the fuzz target {{ report.executable }} to crash. The faulting input SHA256 hash is {{ report.input_sha256 }}
", + "ado_fields": { + "System.AssignedTo": "INSERT_ASSIGNED_HERE", + "System.Tags": "OneFuzz", + "System.AreaPath": "OS\\WDX\\DXP\\WinDev\\Terminal", + "OSG.Watson.Telemetry14DaysInMarketHits": "1", + "System.IterationPath": "OS\\Future", + "Microsoft.VSTS.Common.CustomString01": "{{ job.project }}", + "Microsoft.VSTS.Common.CustomString02": "{{ job.name }}", + "Microsoft.VSTS.Common.CustomString03": "{{ report.minimized_stack_function_lines_sha256}}", + "System.Title": "[Fuzzing] - {{ report.crash_site }}", + "Microsoft.VSTS.CMMI.HowFound": "Security: Fuzzing", + "OSG.SecurityImpact": "Security Triage Requested", + "OSG.SDLSeverity": "Moderate", + "Microsoft.VSTS.TCM.ReproSteps": "The fuzzing target ({{ job.project }} {{ job.name }} {{ job.build }}) reported a crash.
{%if report.asan_log %} AddressSanitizer reported the following details:
 {{ report.asan_log }} 
{% else %} Faulting call stack:
{% endif %} You can reproduce the issue remotely in OneFuzz by running the following command:
 {{ repro_cmd }} 
" + }, + "on_duplicate": { + "set_state": {"Resolved": "Active", "Closed": "Active"}, + "ado_fields": { + "System.IterationPath": "OS\\Future" + }, + "increment": ["OSG.Watson.Telemetry14DaysInMarketHits"] + } + } +} diff --git a/build/pipelines/fuzz.yml b/build/pipelines/fuzz.yml new file mode 100644 index 00000000000..a5580118aa3 --- /dev/null +++ b/build/pipelines/fuzz.yml @@ -0,0 +1,59 @@ +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - docs/* + - samples/* + - tools/* + +pr: none + +# 0.0.yyMM.dd## +# 0.0.1904.0900 +name: 0.0.$(Date:yyMM).$(Date:dd)$(Rev:rr) + +stages: + - stage: Build_Fuzz_Config + displayName: Build Fuzzers + dependsOn: [] + condition: succeeded() + jobs: + - template: ./templates/build-console-fuzzing.yml + parameters: + platform: x64 + - stage: OneFuzz + displayName: Submit OneFuzz Job + dependsOn: ['Build_Fuzz_Config'] + condition: succeeded() + pool: + vmImage: 'ubuntu-latest' + variables: + artifactName: fuzzingBuildOutput + jobs: + - job: + steps: + - task: DownloadBuildArtifacts@0 + inputs: + artifactName: $(artifactName) + downloadPath: $(Build.ArtifactStagingDirectory) + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + addToPath: true + architecture: 'x64' + - bash: | + set -ex + pip -q install onefuzz + onefuzz config --endpoint $(endpoint) --client_id $(client_id) --authority $(authority) --tenant_domain $(tenant_domain) --client_secret $(client_secret) + sed -i s/INSERT_PAT_HERE/$(ado_pat)/ build/Fuzz/notifications-ado.json + sed -i s/INSERT_ASSIGNED_HERE/$(ado_assigned_to)/ build/Fuzz/notifications-ado.json + displayName: Configure OneFuzz + - bash: | + onefuzz template libfuzzer basic --colocate_all_tasks --vm_count 1 --target_exe $target_exe_path --notification_config build/Fuzz/notifications-ado.json OpenConsole $test_name $(Build.SourceVersion) default + displayName: Submit OneFuzz Job + env: + target_exe_path: $(Build.ArtifactStagingDirectory)/$(artifactName)/Fuzzing/x64/test/OpenConsoleFuzzer.exe + test_name: WriteCharsLegacy diff --git a/build/pipelines/templates/build-console-fuzzing.yml b/build/pipelines/templates/build-console-fuzzing.yml new file mode 100644 index 00000000000..1c8dee82cc2 --- /dev/null +++ b/build/pipelines/templates/build-console-fuzzing.yml @@ -0,0 +1,114 @@ +parameters: + configuration: 'Fuzzing' + platform: '' + additionalBuildArguments: '' + +jobs: +- job: Build${{ parameters.platform }}${{ parameters.configuration }} + displayName: Build ${{ parameters.platform }} ${{ parameters.configuration }} + variables: + BuildConfiguration: ${{ parameters.configuration }} + BuildPlatform: ${{ parameters.platform }} + pool: + ${{ if eq(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}: + name: WinDevPoolOSS-L + ${{ if ne(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}: + name: WinDevPool-L + demands: ImageOverride -equals WinDevVS16-latest + + steps: + - checkout: self + submodules: true + clean: true + + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 5.2.0' + inputs: + versionSpec: 5.2.0 + + # In the Microsoft Azure DevOps tenant, NuGetCommand is ambiguous. + # This should be `task: NuGetCommand@2` + - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 + displayName: Restore NuGet packages for solution + inputs: + command: restore + feedsToUse: config + configPath: NuGet.config + restoreSolution: OpenConsole.sln + restoreDirectory: '$(Build.SourcesDirectory)\packages' + + - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 + displayName: Restore NuGet packages for extraneous build actions + inputs: + command: restore + feedsToUse: config + configPath: NuGet.config + restoreSolution: build/packages.config + restoreDirectory: '$(Build.SourcesDirectory)\packages' + + # The environment variable VCToolsInstallDir isn't defined on lab machines, so we need to retrieve it ourselves. + - script: | + "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -Latest -requires Microsoft.Component.MSBuild -property InstallationPath > %TEMP%\vsinstalldir.txt + set /p _VSINSTALLDIR15=<%TEMP%\vsinstalldir.txt + del %TEMP%\vsinstalldir.txt + call "%_VSINSTALLDIR15%\Common7\Tools\VsDevCmd.bat" + echo VCToolsInstallDir = %VCToolsInstallDir% + echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir% + displayName: 'Retrieve VC tools directory' + + - task: VSBuild@1 + displayName: 'Build solution **\OpenConsole.sln' + inputs: + solution: '**\OpenConsole.sln' + vsVersion: 16.0 + platform: '$(BuildPlatform)' + configuration: '$(BuildConfiguration)' + msbuildArgs: "${{ parameters.additionalBuildArguments }}" + clean: true + maximumCpuCount: true + + - task: PowerShell@2 + displayName: 'Rationalize build platform' + inputs: + targetType: inline + script: | + $Arch = "$(BuildPlatform)" + If ($Arch -Eq "x86") { $Arch = "Win32" } + Write-Host "##vso[task.setvariable variable=RationalizedBuildPlatform]${Arch}" + + - task: CopyFiles@2 + displayName: 'Copy result logs to Artifacts' + inputs: + Contents: | + **/*.wtl + **/*onBuildMachineResults.xml + ${{ parameters.testLogPath }} + TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test' + OverWrite: true + flattenFolders: true + + - task: CopyFiles@2 + displayName: 'Copy outputs needed for test runs to Artifacts' + inputs: + Contents: | + $(Build.SourcesDirectory)/bin/$(RationalizedBuildPlatform)/$(BuildConfiguration)/*.exe + $(Build.SourcesDirectory)/bin/$(RationalizedBuildPlatform)/$(BuildConfiguration)/*.dll + $(Build.SourcesDirectory)/bin/$(RationalizedBuildPlatform)/$(BuildConfiguration)/*.xml + **/Microsoft.VCLibs.*.appx + **/TestHostApp/*.exe + **/TestHostApp/*.dll + **/TestHostApp/*.xml + !**/*.pdb + !**/*.ipdb + !**/*.obj + !**/*.pch + TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test' + OverWrite: true + flattenFolders: true + condition: succeeded() + + - task: PublishBuildArtifacts@1 + displayName: 'Publish All Build Artifacts' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'fuzzingBuildOutput' \ No newline at end of file diff --git a/build/pipelines/templates/build-console-steps.yml b/build/pipelines/templates/build-console-steps.yml index 03141ac5f81..bf18b771e40 100644 --- a/build/pipelines/templates/build-console-steps.yml +++ b/build/pipelines/templates/build-console-steps.yml @@ -126,7 +126,7 @@ steps: displayName: 'Publish All Build Artifacts' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'drop' + ArtifactName: 'drop' - task: CopyFiles@2 displayName: 'Copy PGO databases needed for PGO instrumentation run' diff --git a/doc/fuzzing.md b/doc/fuzzing.md new file mode 100644 index 00000000000..5d855f5fd17 --- /dev/null +++ b/doc/fuzzing.md @@ -0,0 +1,60 @@ +# Fuzzing + +## Setting up a fuzzer locally + +OpenConsole can be built with a `Fuzzing` configuration. To set up a fuzzer, you'll need an `LLVMFuzzerTestOneInput` function. This serves as a way for the fuzzer to attach itself and inject tests into your fuzz target. + +To build the fuzzer locally, build the OpenConsole solution in the `Fuzzing` configuration. This should output an executable that runs the fuzzer on the provided test case. In the case of PR #9604, the desired executable is located at `bin\x64\Fuzzing\OpenConsoleFuzzer.exe`. + +### Resources +- [LibFuzzer Docs](https://www.llvm.org/docs/LibFuzzer.html) +- [#9604](https://github.com/microsoft/terminal/pull/9604) + +## Setting up OneFuzz + +OneFuzz allows us to run our fuzzers in CI and be alerted of new bugs found in this endeavor. + +### Installing OneFuzz + +You can download the latest OneFuzz CLI on their [releases page](https://github.com/microsoft/onefuzz/releases). + +### Configuring OneFuzz + +To run OneFuzz locally, you'll need to configure its endpoint, client ID, and client secret. Windows has a preset configuration available; this can be found at [this tutorial](https://www.osgwiki.com/wiki/Fuzzing_Service_-_Azure_Edge_and_Platform#Configure_OneFuzz_CLI) on osgwiki. + + + +`onefuzz config --endpoint $(endpoint) --client_id $(client_id) --authority $(authority) --tenant_domain $(tenant_domain)` + +**NOTE**: Our pipeline is already set up with these variables, so you don't need to worry about this when running this on Azure DevOps. + +### Running a job on OneFuzz + +You should now be able to run a job using the following command: + +`onefuzz template libfuzzer basic --target_exe ` + +- `project`: the name of the project +- `name`: the name of the test +- `build`: the identifier for the build (i.e. commit SHA1) +- `pool`: the VM pool to run this on +- `exe_path`: the fuzzer executable output from building your project + +This should also output more information (i.e. job ID) about the newly created job in a JSON format. + +### Enabling notifications + +**NOTE**: Our pipeline is already set up with this functionality. However, here is a quick guide on how to get it set up and modify it to our liking. + +OneFuzz supports multiple notification systems at once including MS Teams and Azure DevOps. See the resources below to learn more about setting these up. + +Our pipeline has been set up to create Azure DevOps work items. + +### Resources +- [OneFuzz GitHub](https://github.com/microsoft/onefuzz) + - [Getting started using OneFuzz](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md) + - [Releases Page](https://github.com/microsoft/onefuzz/releases) +- [Notifications](https://github.com/microsoft/onefuzz/blob/main/docs/notifications.md) + - [MS Teams](https://github.com/microsoft/onefuzz/blob/main/docs/notifications/teams.md) + - [Azure DevOps](https://github.com/microsoft/onefuzz/blob/main/docs/notifications/ado.md) +- [OSG Wiki - OneFuzz](https://www.osgwiki.com/wiki/Fuzzing_Service_-_Azure_Edge_and_Platform) \ No newline at end of file diff --git a/src/host/ft_fuzzer/Host.FuzzWrapper.vcxproj b/src/host/ft_fuzzer/Host.FuzzWrapper.vcxproj index 8486a9e4894..ac5ad0884f5 100644 --- a/src/host/ft_fuzzer/Host.FuzzWrapper.vcxproj +++ b/src/host/ft_fuzzer/Host.FuzzWrapper.vcxproj @@ -82,7 +82,7 @@ - clang_rt.fuzzer-$(OCClangArchitectureName).lib;%(AdditionalDependencies) + WinMM.Lib;clang_rt.fuzzer_MT-$(OCClangArchitectureName).lib;%(AdditionalDependencies) diff --git a/src/host/proxy/Host.Proxy.vcxproj b/src/host/proxy/Host.Proxy.vcxproj index ed7283eba0a..c6129bf828f 100644 --- a/src/host/proxy/Host.Proxy.vcxproj +++ b/src/host/proxy/Host.Proxy.vcxproj @@ -7,6 +7,16 @@ OpenConsoleProxy OpenConsoleProxy DynamicLibrary + + StaticLibrary