Skip to content

Commit 7d044e0

Browse files
nikpivkinnikpivkin
committed
feat(misconf): loading embedded checks as a fallback
1 parent cfddfb3 commit 7d044e0

File tree

8 files changed

+159
-37
lines changed

8 files changed

+159
-37
lines changed

‎pkg/cloud/report/convert.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/aws/aws-sdk-go-v2/aws/arn"
99

1010
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
11+
"github.com/aquasecurity/trivy/pkg/iac/rego"
1112
"github.com/aquasecurity/trivy/pkg/iac/scan"
1213
"github.com/aquasecurity/trivy/pkg/types"
1314
)
@@ -57,7 +58,7 @@ func ConvertResults(results scan.Results, provider string, scoped []string) map[
5758

5859
// empty namespace implies a go rule from defsec, "builtin" refers to a built-in rego rule
5960
// this ensures we don't generate bad links for custom policies
60-
if result.RegoNamespace() == "" || strings.HasPrefix(result.RegoNamespace(), "builtin.") {
61+
if result.RegoNamespace() == "" || rego.IsBuiltinNamespace(result.RegoNamespace()) {
6162
primaryURL = fmt.Sprintf("https://avd.aquasec.com/misconfig/%s", strings.ToLower(result.Rule().AVDID))
6263
}
6364

‎pkg/commands/artifact/run.go‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/aquasecurity/trivy/pkg/fanal/cache"
2121
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
2222
"github.com/aquasecurity/trivy/pkg/flag"
23+
"github.com/aquasecurity/trivy/pkg/iac/rego"
2324
"github.com/aquasecurity/trivy/pkg/javadb"
2425
"github.com/aquasecurity/trivy/pkg/log"
2526
"github.com/aquasecurity/trivy/pkg/misconf"
@@ -49,11 +50,6 @@ const (
4950
)
5051

5152
var (
52-
defaultPolicyNamespaces = []string{
53-
"appshield",
54-
"defsec",
55-
"builtin",
56-
}
5753
SkipScan = errors.New("skip subsequent processes")
5854
)
5955

@@ -597,7 +593,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
597593
configScannerOptions = misconf.ScannerOption{
598594
Debug: opts.Debug,
599595
Trace: opts.Trace,
600-
Namespaces: append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
596+
Namespaces: append(opts.PolicyNamespaces, rego.BuiltinNamespaces()...),
601597
PolicyPaths: append(opts.PolicyPaths, downloadedPolicyPaths...),
602598
DataPaths: opts.DataPaths,
603599
HelmValues: opts.HelmValues,

‎pkg/iac/rego/embed.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88

99
"github.com/open-policy-agent/opa/ast"
1010

11-
rules2 "github.com/aquasecurity/trivy-policies"
11+
checks "github.com/aquasecurity/trivy-policies"
1212
"github.com/aquasecurity/trivy/pkg/iac/rules"
1313
)
1414

@@ -62,11 +62,11 @@ func RegisterRegoRules(modules map[string]*ast.Module) {
6262
}
6363

6464
func LoadEmbeddedPolicies() (map[string]*ast.Module, error) {
65-
return LoadPoliciesFromDirs(rules2.EmbeddedPolicyFileSystem, ".")
65+
return LoadPoliciesFromDirs(checks.EmbeddedPolicyFileSystem, ".")
6666
}
6767

6868
func LoadEmbeddedLibraries() (map[string]*ast.Module, error) {
69-
return LoadPoliciesFromDirs(rules2.EmbeddedLibraryFileSystem, ".")
69+
return LoadPoliciesFromDirs(checks.EmbeddedLibraryFileSystem, ".")
7070
}
7171

7272
func LoadPoliciesFromDirs(target fs.FS, paths ...string) (map[string]*ast.Module, error) {

‎pkg/iac/rego/load.go‎

Lines changed: 93 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,30 @@ import (
55
"fmt"
66
"io"
77
"io/fs"
8+
"path/filepath"
89
"strings"
910

1011
"github.com/open-policy-agent/opa/ast"
1112
"github.com/open-policy-agent/opa/bundle"
13+
"github.com/samber/lo"
1214
)
1315

16+
var builtinNamespaces = map[string]struct{}{
17+
"builtin": {},
18+
"defsec": {},
19+
"appshield": {},
20+
}
21+
22+
func BuiltinNamespaces() []string {
23+
return lo.Keys(builtinNamespaces)
24+
}
25+
26+
func IsBuiltinNamespace(namespace string) bool {
27+
return lo.ContainsBy(BuiltinNamespaces(), func(ns string) bool {
28+
return strings.HasPrefix(namespace, ns+".")
29+
})
30+
}
31+
1432
func IsRegoFile(name string) bool {
1533
return strings.HasSuffix(name, bundle.RegoExt) && !strings.HasSuffix(name, "_test"+bundle.RegoExt)
1634
}
@@ -38,28 +56,20 @@ func (s *Scanner) loadPoliciesFromReaders(readers []io.Reader) (map[string]*ast.
3856
return modules, nil
3957
}
4058

41-
func (s *Scanner) loadEmbedded(enableEmbeddedLibraries, enableEmbeddedPolicies bool) error {
42-
if enableEmbeddedLibraries {
43-
loadedLibs, errLoad := LoadEmbeddedLibraries()
44-
if errLoad != nil {
45-
return fmt.Errorf("failed to load embedded rego libraries: %w", errLoad)
46-
}
47-
for name, policy := range loadedLibs {
48-
s.policies[name] = policy
49-
}
50-
s.debug.Log("Loaded %d embedded libraries.", len(loadedLibs))
59+
func (s *Scanner) loadEmbedded() error {
60+
loaded, err := LoadEmbeddedLibraries()
61+
if err != nil {
62+
return fmt.Errorf("failed to load embedded rego libraries: %w", err)
5163
}
64+
s.embeddedLibs = loaded
65+
s.debug.Log("Loaded %d embedded libraries.", len(loaded))
5266

53-
if enableEmbeddedPolicies {
54-
loaded, err := LoadEmbeddedPolicies()
55-
if err != nil {
56-
return fmt.Errorf("failed to load embedded rego policies: %w", err)
57-
}
58-
for name, policy := range loaded {
59-
s.policies[name] = policy
60-
}
61-
s.debug.Log("Loaded %d embedded policies.", len(loaded))
67+
loaded, err = LoadEmbeddedPolicies()
68+
if err != nil {
69+
return fmt.Errorf("failed to load embedded rego policies: %w", err)
6270
}
71+
s.embeddedChecks = loaded
72+
s.debug.Log("Loaded %d embedded policies.", len(loaded))
6373

6474
return nil
6575
}
@@ -75,7 +85,7 @@ func (s *Scanner) LoadPolicies(enableEmbeddedLibraries, enableEmbeddedPolicies b
7585
srcFS = s.policyFS
7686
}
7787

78-
if err := s.loadEmbedded(enableEmbeddedLibraries, enableEmbeddedPolicies); err != nil {
88+
if err := s.loadEmbedded(); err != nil {
7989
return err
8090
}
8191

@@ -124,9 +134,71 @@ func (s *Scanner) LoadPolicies(enableEmbeddedLibraries, enableEmbeddedPolicies b
124134
}
125135
s.store = store
126136

137+
if enableEmbeddedPolicies {
138+
s.policies = lo.Assign(s.policies, s.embeddedChecks)
139+
}
140+
141+
if enableEmbeddedLibraries {
142+
s.policies = lo.Assign(s.policies, s.embeddedLibs)
143+
}
144+
127145
return s.compilePolicies(srcFS, paths)
128146
}
129147

148+
func (s *Scanner) fallbackChecks(compiler *ast.Compiler) {
149+
150+
var excludedFiles []string
151+
152+
for _, e := range compiler.Errors {
153+
if _, ok := e.Details.(*ast.RefErrInvalidDetail); !ok {
154+
continue
155+
}
156+
157+
loc := e.Location.File
158+
159+
if lo.Contains(excludedFiles, loc) {
160+
continue
161+
}
162+
163+
badPolicy, exists := s.policies[loc]
164+
if !exists || badPolicy == nil {
165+
continue
166+
}
167+
168+
if !IsBuiltinNamespace(getModuleNamespace(badPolicy)) {
169+
continue
170+
}
171+
172+
s.debug.Log("Error occurred while parsing: %s, %s. \nTry loading embedded policy.", loc, e.Error())
173+
174+
embedded := s.findMatchedEmbeddedCheck(badPolicy)
175+
if embedded == nil {
176+
s.debug.Log("Failed to find embedded policy: %s", loc)
177+
continue
178+
}
179+
180+
s.debug.Log("Found embedded policy: %s", embedded.Package.Location.File)
181+
delete(s.policies, loc) // remove bad policy
182+
s.policies[embedded.Package.Location.File] = embedded
183+
delete(s.embeddedChecks, embedded.Package.Location.File) // avoid infinite loop if embedded check contains ref error
184+
excludedFiles = append(excludedFiles, e.Location.File)
185+
}
186+
187+
compiler.Errors = lo.Filter(compiler.Errors, func(e *ast.Error, _ int) bool {
188+
return !lo.Contains(excludedFiles, e.Location.File)
189+
})
190+
}
191+
192+
func (s *Scanner) findMatchedEmbeddedCheck(module *ast.Module) *ast.Module {
193+
for _, policy := range s.embeddedChecks {
194+
if policy.Package.Path.String() == module.Package.Path.String() ||
195+
filepath.Base(policy.Package.Location.File) == filepath.Base(module.Package.Location.File) {
196+
return policy
197+
}
198+
}
199+
return nil
200+
}
201+
130202
func (s *Scanner) prunePoliciesWithError(compiler *ast.Compiler) error {
131203
if len(compiler.Errors) > s.regoErrorLimit {
132204
s.debug.Log("Error(s) occurred while loading policies")
@@ -157,6 +229,7 @@ func (s *Scanner) compilePolicies(srcFS fs.FS, paths []string) error {
157229

158230
compiler.Compile(s.policies)
159231
if compiler.Failed() {
232+
s.fallbackChecks(compiler)
160233
if err := s.prunePoliciesWithError(compiler); err != nil {
161234
return err
162235
}

‎pkg/iac/rego/load_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"bytes"
55
"embed"
66
"testing"
7+
"testing/fstest"
78

9+
trivy_policies "github.com/aquasecurity/trivy-policies"
10+
"github.com/aquasecurity/trivy/pkg/iac/scanners/options"
811
"github.com/aquasecurity/trivy/pkg/iac/types"
912
"github.com/stretchr/testify/assert"
1013
"github.com/stretchr/testify/require"
@@ -13,6 +16,9 @@ import (
1316
//go:embed all:testdata/policies
1417
var testEmbedFS embed.FS
1518

19+
//go:embed testdata/embedded
20+
var embeddedPoliciesFS embed.FS
21+
1622
func Test_RegoScanning_WithSomeInvalidPolicies(t *testing.T) {
1723
t.Run("allow no errors", func(t *testing.T) {
1824
var debugBuf bytes.Buffer
@@ -42,5 +48,38 @@ func Test_RegoScanning_WithSomeInvalidPolicies(t *testing.T) {
4248

4349
assert.Contains(t, debugBuf.String(), "Error occurred while parsing: testdata/policies/invalid.rego, testdata/policies/invalid.rego:7")
4450
})
51+
}
52+
53+
func Test_FallbackToEmbedded(t *testing.T) {
54+
scanner := NewScanner(
55+
types.SourceDockerfile,
56+
options.ScannerWithRegoErrorLimits(0),
57+
)
58+
fsys := fstest.MapFS{
59+
"policies/my-policy2.rego": &fstest.MapFile{
60+
Data: []byte(`# METADATA
61+
# schemas:
62+
# - input: schema["fooschema"]
63+
64+
package builtin.test
4565
66+
deny {
67+
input.evil == "foo bar"
68+
}`),
69+
},
70+
"schemas/fooschema.json": &fstest.MapFile{
71+
Data: []byte(`{
72+
"$schema": "http://json-schema.org/draft-07/schema#",
73+
"type": "object",
74+
"properties": {
75+
"foo": {
76+
"type": "string"
77+
}
78+
}
79+
}`),
80+
},
81+
}
82+
trivy_policies.EmbeddedPolicyFileSystem = embeddedPoliciesFS
83+
err := scanner.LoadPolicies(false, false, fsys, []string{"."}, nil)
84+
assert.NoError(t, err)
4685
}

‎pkg/iac/rego/scanner.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"io"
99
"io/fs"
10+
"maps"
1011
"strings"
1112

1213
"github.com/open-policy-agent/opa/ast"
@@ -41,6 +42,9 @@ type Scanner struct {
4142
spec string
4243
inputSchema interface{} // unmarshalled into this from a json schema document
4344
sourceType types.Source
45+
46+
embeddedLibs map[string]*ast.Module
47+
embeddedChecks map[string]*ast.Module
4448
}
4549

4650
func (s *Scanner) SetUseEmbeddedLibraries(b bool) {
@@ -135,13 +139,12 @@ func NewScanner(source types.Source, opts ...options.ScannerOption) *Scanner {
135139
s := &Scanner{
136140
regoErrorLimit: ast.CompileErrorLimitDefault,
137141
sourceType: source,
138-
ruleNamespaces: map[string]struct{}{
139-
"builtin": {},
140-
"appshield": {},
141-
"defsec": {},
142-
},
143-
runtimeValues: addRuntimeValues(),
142+
ruleNamespaces: make(map[string]struct{}),
143+
runtimeValues: addRuntimeValues(),
144144
}
145+
146+
maps.Copy(s.ruleNamespaces, builtinNamespaces)
147+
145148
for _, opt := range opts {
146149
opt(s)
147150
}

‎pkg/iac/rego/testdata/embedded/my-policy1.rego‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# METADATA
2+
# schemas:
3+
# - input: schema["fooschema"]
4+
5+
package builtin.test
6+
7+
deny {
8+
input.foo == "foo bar"
9+
}

‎pkg/scanner/local/scan.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
1818
"github.com/aquasecurity/trivy/pkg/fanal/applier"
1919
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
20+
"github.com/aquasecurity/trivy/pkg/iac/rego"
2021
"github.com/aquasecurity/trivy/pkg/licensing"
2122
"github.com/aquasecurity/trivy/pkg/log"
2223
"github.com/aquasecurity/trivy/pkg/scanner/langpkg"
@@ -383,7 +384,7 @@ func toDetectedMisconfiguration(res ftypes.MisconfResult, defaultSeverity dbType
383384

384385
// empty namespace implies a go rule from defsec, "builtin" refers to a built-in rego rule
385386
// this ensures we don't generate bad links for custom policies
386-
if res.Namespace == "" || strings.HasPrefix(res.Namespace, "builtin.") {
387+
if res.Namespace == "" || rego.IsBuiltinNamespace(res.Namespace) {
387388
primaryURL = fmt.Sprintf("https://avd.aquasec.com/misconfig/%s", strings.ToLower(res.ID))
388389
res.References = append(res.References, primaryURL)
389390
}

0 commit comments

Comments
 (0)