Skip to content

Commit

Permalink
✨ [experimental] Probe support for fuzzing check (#3230)
Browse files Browse the repository at this point in the history 
* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* update

Signed-off-by: laurentsimon <[email protected]>

* comments

Signed-off-by: laurentsimon <[email protected]>

* unit tests and linter

Signed-off-by: laurentsimon <[email protected]>

* remove raw from check request in e2e tests

Signed-off-by: laurentsimon <[email protected]>

* remove redundant finding check

Signed-off-by: laurentsimon <[email protected]>

* typo

Signed-off-by: laurentsimon <[email protected]>

* adress comments

Signed-off-by: laurentsimon <[email protected]>

---------

Signed-off-by: laurentsimon <[email protected]>
  • Loading branch information
@laurentsimon
laurentsimon authored Aug 21, 2023
1 parent 7f64da7 commit d177169
Show file tree
Hide file tree
Showing 30 changed files with 1,858 additions and 72 deletions.
49 changes: 48 additions & 1 deletion checks/evaluation/dependency_update_tool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ func TestDependencyUpdateTool(t *testing.T) {
Probe: "toolSonatypeLiftInstalled",
Outcome: finding.OutcomePositive,
},
{
Probe: "toolRenovateInstalled",
Outcome: finding.OutcomeNegative,
},
},
want: checker.CheckResult{
Score: 10,
Expand Down Expand Up @@ -153,7 +157,50 @@ func TestDependencyUpdateTool(t *testing.T) {
},
},
{
name: "empty tool list",
name: "missing probes renovate",
findings: []finding.Finding{
{
Probe: "toolDependabotInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolPyUpInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolSonatypeInstalled",
Outcome: finding.OutcomeNegative,
},
},
err: true,
want: checker.CheckResult{
Score: -1,
},
},
{
name: "invalid probe name",
findings: []finding.Finding{
{
Probe: "toolDependabotInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolRenovateInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolPyUpInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolSonatypeInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "toolInvalidProbeName",
Outcome: finding.OutcomeNegative,
},
},
want: checker.CheckResult{
Score: -1,
},
Expand Down
54 changes: 29 additions & 25 deletions checks/evaluation/fuzzing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,44 @@
package evaluation

import (
"fmt"

"github.com/ossf/scorecard/v4/checker"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
)

// Fuzzing applies the score policy for the Fuzzing check.
func Fuzzing(name string, dl checker.DetailLogger,
r *checker.FuzzingData,
func Fuzzing(name string,
findings []finding.Finding,
) checker.CheckResult {
if r == nil {
e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
return checker.CreateRuntimeErrorResult(name, e)
// We have 7 unique probes, each should have a finding.
expectedProbes := []string{
fuzzedWithClusterFuzzLite.Probe,
fuzzedWithGoNative.Probe,
fuzzedWithOneFuzz.Probe,
fuzzedWithOSSFuzz.Probe,
fuzzedWithPropertyBasedHaskell.Probe,
fuzzedWithPropertyBasedJavascript.Probe,
fuzzedWithPropertyBasedTypescript.Probe,
}

if len(r.Fuzzers) == 0 {
return checker.CreateMinScoreResult(name, "project is not fuzzed")
if !finding.UniqueProbesEqual(findings, expectedProbes) {
e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
return checker.CreateRuntimeErrorResult(name, e)
}
fuzzers := []string{}
for i := range r.Fuzzers {
fuzzer := r.Fuzzers[i]
for _, f := range fuzzer.Files {
msg := checker.LogMessage{
Path: f.Path,
Type: f.Type,
Offset: f.Offset,
}
if f.Snippet != "" {
msg.Text = f.Snippet
}
dl.Info(&msg)

// Compute the score.
for i := range findings {
f := &findings[i]
if f.Outcome == finding.OutcomePositive {
return checker.CreateMaxScoreResult(name, "project is fuzzed")
}
fuzzers = append(fuzzers, fuzzer.Name)
}
return checker.CreateMaxScoreResult(name,
fmt.Sprintf("project is fuzzed with %v", fuzzers))
return checker.CreateMinScoreResult(name, "project is not fuzzed")
}
165 changes: 138 additions & 27 deletions checks/evaluation/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"

"github.com/ossf/scorecard/v4/checker"
scut "github.com/ossf/scorecard/v4/utests"
"github.com/ossf/scorecard/v4/finding"
)

func TestFuzzing(t *testing.T) {
t.Parallel()
type args struct { //nolint
name string
dl checker.DetailLogger
r *checker.FuzzingData
name string
findings []finding.Finding
}
tests := []struct {
name string
Expand All @@ -39,8 +38,36 @@ func TestFuzzing(t *testing.T) {
name: "Fuzzing - no fuzzing",
args: args{
name: "Fuzzing",
dl: &scut.TestDetailLogger{},
r: &checker.FuzzingData{},
findings: []finding.Finding{
{
Probe: "fuzzedWithClusterFuzzLite",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithGoNative",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOSSFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedHaskell",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedJavascript",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedTypescript",
Outcome: finding.OutcomeNegative,
},
},
},
want: checker.CheckResult{
Score: 0,
Expand All @@ -50,53 +77,137 @@ func TestFuzzing(t *testing.T) {
},
},
{
name: "Fuzzing - fuzzing",
name: "Fuzzing - fuzzing GoNative",
args: args{
name: "Fuzzing",
dl: &scut.TestDetailLogger{},
r: &checker.FuzzingData{
Fuzzers: []checker.Tool{
{
Name: "Fuzzing",
Files: []checker.File{
{
Path: "Fuzzing",
Type: 0,
Offset: 1,
Snippet: "Fuzzing",
},
},
},
findings: []finding.Finding{
{
Probe: "fuzzedWithClusterFuzzLite",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithGoNative",
Outcome: finding.OutcomePositive,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOSSFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedHaskell",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedJavascript",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedTypescript",
Outcome: finding.OutcomeNegative,
},
},
},
want: checker.CheckResult{
Score: 10,
Name: "Fuzzing",
Version: 2,
Reason: "project is fuzzed with [Fuzzing]",
Reason: "project is fuzzed",
},
},
{
name: "Fuzzing - fuzzing data nil",
name: "Fuzzing - fuzzing missing GoNative finding",
args: args{
name: "Fuzzing",
dl: &scut.TestDetailLogger{},
r: nil,
findings: []finding.Finding{
{
Probe: "fuzzedWithClusterFuzzLite",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOSSFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedHaskell",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedJavascript",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedTypescript",
Outcome: finding.OutcomeNegative,
},
},
},
want: checker.CheckResult{
Score: -1,
Name: "Fuzzing",
Version: 2,
Reason: "internal error: invalid probe results",
},
},
{
name: "Fuzzing - fuzzing invalid probe name",
args: args{
name: "Fuzzing",
findings: []finding.Finding{
{
Probe: "fuzzedWithClusterFuzzLite",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithGoNative",
Outcome: finding.OutcomePositive,
},
{
Probe: "fuzzedWithOneFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithOSSFuzz",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedHaskell",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedJavascript",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithPropertyBasedTypescript",
Outcome: finding.OutcomeNegative,
},
{
Probe: "fuzzedWithInvalidProbeName",
Outcome: finding.OutcomePositive,
},
},
},
want: checker.CheckResult{
Score: -1,
Name: "Fuzzing",
Version: 2,
Reason: "internal error: empty raw data",
Reason: "internal error: invalid probe results",
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := Fuzzing(tt.args.name, tt.args.dl, tt.args.r); !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error")) { //nolint:lll
if got := Fuzzing(tt.args.name, tt.args.findings); !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error")) { //nolint:lll
t.Errorf("Fuzzing() = %v, want %v", got, cmp.Diff(got, tt.want, cmpopts.IgnoreFields(checker.CheckResult{}, "Error"))) //nolint:lll
}
})
Expand Down
14 changes: 11 additions & 3 deletions checks/fuzzing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/ossf/scorecard/v4/checks/evaluation"
"github.com/ossf/scorecard/v4/checks/raw"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/probes"
)

// CheckFuzzing is the registered name for Fuzzing.
Expand All @@ -41,9 +42,16 @@ func Fuzzing(c *checker.CheckRequest) checker.CheckResult {
}

// Set the raw results.
if c.RawResults != nil {
c.RawResults.FuzzingResults = rawData
pRawResults := getRawResults(c)
pRawResults.FuzzingResults = rawData

// Evaluate the probes.
findings, err := evaluateProbes(c, pRawResults, probes.Fuzzing)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
return checker.CreateRuntimeErrorResult(CheckFuzzing, e)
}

return evaluation.Fuzzing(CheckFuzzing, c.Dlogger, &rawData)
// Return the score evaluation.
return evaluation.Fuzzing(CheckFuzzing, findings)
}
Loading

0 comments on commit d177169

Please sign in to comment.