Skip to content

Commit ef44a59

Browse files
cx-miguel-silvacx-artur-ribeiro
cx-miguel-silva
and
cx-artur-ribeiro
authored
feat(resolver): kubernetes circular dependency is causing resource exhaustion (#7421)
* stage * draft check for circular references * comment failing test and solve lint * remove stashed currentResolutionPath from previous resolved files allowing once resolutions * remove wrong comments and add no lint statement * fix TestResolver_Resolve_Ansible_Vars unit test * fix 2 tests on TestResolver_Resolve_With_ResolveReferences and 1 test on TestResolver_Resolve_Without_ResolveReferences (5/8) * fix all unit tests * add lint ignore * remove lint ignore * add Test_checkCircularReference tests * remove deprecated function ioutil.ReadFile * remove unnecessary comments * first json temporary solution and handleMap RefMedatada refactor * add resolution for json files * solve unit tests although they seem to be wrongly implementedgi * refactor code and solve unit tests * fix lint issues and e2e test * add e2e that compares the payload between json and yaml files * fix mixing want status and lint lll * add missing payloads * refactor payload, folder name and fix e2e * change comments on e2e * refactor names to be according to other files naming conventions * remove duplicated unit test * fix typo on e2e comment * remove resolved file at least once solution * update e2e 099 payload with new implemented code * update file and id from e2e payload * remove currentResolutionPath from TestResolver * revert addition of \n to test error output * add Clear mechanism on ResolvedFilesCache to force re-resolution of all files * add nolint gocyclo * check if the file can or cannot be cached during the for cycle --------- Co-authored-by: ArturRibeiro-CX <[email protected]> Co-authored-by: ArturRibeiro-CX <[email protected]>
1 parent 622cc10 commit ef44a59

File tree

14 files changed

+733
-211
lines changed

14 files changed

+733
-211
lines changed

‎e2e/fixtures/E2E_CLI_099_JSON_PAYLOAD.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"document": [
3+
{
4+
"components": {
5+
"schemas": {
6+
"Author": {
7+
"RefMetadata": {
8+
"$ref": "./components/schemas/Author.json",
9+
"alone": true
10+
},
11+
"properties": {
12+
"books": {
13+
"items": {
14+
"RefMetadata": {
15+
"$ref": "Book.json",
16+
"alone": true
17+
},
18+
"properties": {
19+
"author": {
20+
"RefMetadata": {
21+
"$ref": "Author.json",
22+
"alone": true
23+
}
24+
},
25+
"books": {
26+
"items": {
27+
"RefMetadata": {
28+
"$ref": "Book.json",
29+
"alone": true
30+
}
31+
},
32+
"type": "array"
33+
},
34+
"id": {
35+
"type": "integer"
36+
},
37+
"title": {
38+
"type": "string"
39+
}
40+
},
41+
"type": "object"
42+
},
43+
"type": "array"
44+
},
45+
"id": {
46+
"type": "integer"
47+
},
48+
"name": {
49+
"type": "string"
50+
}
51+
},
52+
"type": "object"
53+
},
54+
"Book": {
55+
"RefMetadata": {
56+
"$ref": "./components/schemas/Book.json",
57+
"alone": true
58+
},
59+
"properties": {
60+
"author": {
61+
"RefMetadata": {
62+
"$ref": "Author.json",
63+
"alone": true
64+
},
65+
"properties": {
66+
"books": {
67+
"items": {
68+
"RefMetadata": {
69+
"$ref": "Book.json",
70+
"alone": true
71+
}
72+
},
73+
"type": "array"
74+
},
75+
"id": {
76+
"type": "integer"
77+
},
78+
"name": {
79+
"type": "string"
80+
}
81+
},
82+
"type": "object"
83+
},
84+
"books": {
85+
"items": {
86+
"RefMetadata": {
87+
"$ref": "Book.json",
88+
"alone": true
89+
}
90+
},
91+
"type": "array"
92+
},
93+
"id": {
94+
"type": "integer"
95+
},
96+
"title": {
97+
"type": "string"
98+
}
99+
},
100+
"type": "object"
101+
}
102+
}
103+
},
104+
"file": "file",
105+
"id": "0",
106+
"info": {
107+
"title": "Complex Cyclic References API",
108+
"version": "1.0.0"
109+
},
110+
"openapi": "3.0.0",
111+
"paths": {}
112+
}
113+
]
114+
}

‎e2e/fixtures/E2E_CLI_099_YAML_PAYLOAD.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"document": [
3+
{
4+
"components": {
5+
"schemas": {
6+
"Author": {
7+
"RefMetadata": {
8+
"$ref": "./components/schemas/Author.yaml",
9+
"alone": true
10+
},
11+
"properties": {
12+
"books": {
13+
"items": {
14+
"RefMetadata": {
15+
"$ref": "Book.yaml",
16+
"alone": true
17+
},
18+
"properties": {
19+
"author": {
20+
"RefMetadata": {
21+
"$ref": "Author.yaml",
22+
"alone": true
23+
}
24+
},
25+
"books": {
26+
"items": {
27+
"RefMetadata": {
28+
"$ref": "Book.yaml",
29+
"alone": true
30+
}
31+
},
32+
"type": "array"
33+
},
34+
"id": {
35+
"type": "integer"
36+
},
37+
"title": {
38+
"type": "string"
39+
}
40+
},
41+
"type": "object"
42+
},
43+
"type": "array"
44+
},
45+
"id": {
46+
"type": "integer"
47+
},
48+
"name": {
49+
"type": "string"
50+
}
51+
},
52+
"type": "object"
53+
},
54+
"Book": {
55+
"RefMetadata": {
56+
"$ref": "./components/schemas/Book.yaml",
57+
"alone": true
58+
},
59+
"properties": {
60+
"author": {
61+
"RefMetadata": {
62+
"$ref": "Author.yaml",
63+
"alone": true
64+
},
65+
"properties": {
66+
"books": {
67+
"items": {
68+
"RefMetadata": {
69+
"$ref": "Book.yaml",
70+
"alone": true
71+
}
72+
},
73+
"type": "array"
74+
},
75+
"id": {
76+
"type": "integer"
77+
},
78+
"name": {
79+
"type": "string"
80+
}
81+
},
82+
"type": "object"
83+
},
84+
"books": {
85+
"items": {
86+
"RefMetadata": {
87+
"$ref": "Book.yaml",
88+
"alone": true
89+
}
90+
},
91+
"type": "array"
92+
},
93+
"id": {
94+
"type": "integer"
95+
},
96+
"title": {
97+
"type": "string"
98+
}
99+
},
100+
"type": "object"
101+
}
102+
}
103+
},
104+
"file": "file",
105+
"id": "0",
106+
"info": {
107+
"title": "Complex Cyclic References API",
108+
"version": "1.0.0"
109+
},
110+
"openapi": "3.0.0",
111+
"paths": {}
112+
}
113+
]
114+
}

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIJson/components/schemas/Author.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"type": "object",
3+
"properties": {
4+
"id": {
5+
"type": "integer"
6+
},
7+
"name": {
8+
"type": "string"
9+
},
10+
"books": {
11+
"type": "array",
12+
"items": {
13+
"$ref": "Book.json"
14+
}
15+
}
16+
}
17+
}

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIJson/components/schemas/Book.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "object",
3+
"properties": {
4+
"id": {
5+
"type": "integer"
6+
},
7+
"title": {
8+
"type": "string"
9+
},
10+
"author": {
11+
"$ref": "Author.json"
12+
},
13+
"books": {
14+
"type": "array",
15+
"items": {
16+
"$ref": "Book.json"
17+
}
18+
}
19+
}
20+
}

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIJson/openAPI.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"openapi": "3.0.0",
3+
"info": {
4+
"title": "Complex Cyclic References API",
5+
"version": "1.0.0"
6+
},
7+
"paths": {},
8+
"components": {
9+
"schemas": {
10+
"Author": {
11+
"$ref": "./components/schemas/Author.json"
12+
},
13+
"Book": {
14+
"$ref": "./components/schemas/Book.json"
15+
}
16+
}
17+
}
18+
}

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIYaml/components/schemas/Author.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
type: object
2+
properties:
3+
id:
4+
type: integer
5+
name:
6+
type: string
7+
books:
8+
type: array
9+
items:
10+
$ref: "Book.yaml"

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIYaml/components/schemas/Book.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
type: object
2+
properties:
3+
id:
4+
type: integer
5+
title:
6+
type: string
7+
author:
8+
$ref: "Author.yaml"
9+
books:
10+
type: array
11+
items:
12+
$ref: "Book.yaml"

‎e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIYaml/openAPI.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
openapi: 3.0.0
2+
info:
3+
title: Complex Cyclic References API
4+
version: 1.0.0
5+
paths: {}
6+
components:
7+
schemas:
8+
Author:
9+
$ref: "./components/schemas/Author.yaml"
10+
Book:
11+
$ref: "./components/schemas/Book.yaml"

‎e2e/testcases/e2e-cli-099_compare_openapi_payload_json_yaml.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Package testcases provides end-to-end (E2E) testing functionality for the application.
2+
package testcases
3+
4+
// E2E-CLI-099 - KICS scan with OpenAPI reference resolution enabled on JSON and YAML files containing circular references.
5+
// The scan should complete successfully, returning exit code 50, producing equivalent payloads for both formats.
6+
// The only differences should be the file extensions between payloads.
7+
func init() { //nolint
8+
testSample := TestCase{
9+
Name: "scan should generate equivalent payloads for OpenAPI YAML and JSON files with circular references [E2E-CLI-099]",
10+
Args: args{
11+
Args: []cmdArgs{
12+
[]string{
13+
"scan", "-p", "\"/path/e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIJson/openAPI.json\"",
14+
"-v", "-d", "/path/e2e/output/E2E_CLI_099_JSON_PAYLOAD.json", "--enable-openapi-refs",
15+
},
16+
[]string{
17+
"scan", "-p", "\"/path/e2e/fixtures/samples/compare-openapi-payload-json-yaml/openAPIYaml/openAPI.yaml\"",
18+
"-v", "-d", "/path/e2e/output/E2E_CLI_099_YAML_PAYLOAD.json", "--enable-openapi-refs",
19+
},
20+
},
21+
ExpectedPayload: []string{
22+
"E2E_CLI_099_JSON_PAYLOAD.json",
23+
"E2E_CLI_099_YAML_PAYLOAD.json",
24+
},
25+
},
26+
WantStatus: []int{50, 50},
27+
}
28+
29+
Tests = append(Tests, testSample)
30+
}

‎pkg/engine/provider/filesystem.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ func ignoreDamagedFiles(path string) bool {
123123
func (s *FileSystemSourceProvider) GetSources(ctx context.Context,
124124
extensions model.Extensions, sink Sink, resolverSink ResolverSink) error {
125125
for _, scanPath := range s.paths {
126-
resolved := false
127126
fileInfo, err := os.Stat(scanPath)
128127
if err != nil {
129128
return errors.Wrap(err, "failed to open path")
@@ -132,7 +131,7 @@ func (s *FileSystemSourceProvider) GetSources(ctx context.Context,
132131
if !fileInfo.IsDir() {
133132
c, openFileErr := openScanFile(scanPath, extensions)
134133
if openFileErr != nil {
135-
if openFileErr == ErrNotSupportedFile || ignoreDamagedFiles(scanPath) {
134+
if errors.Is(openFileErr, ErrNotSupportedFile) || ignoreDamagedFiles(scanPath) {
136135
continue
137136
}
138137
return openFileErr
@@ -143,7 +142,7 @@ func (s *FileSystemSourceProvider) GetSources(ctx context.Context,
143142
continue
144143
}
145144

146-
err = s.walkDir(ctx, scanPath, resolved, sink, resolverSink, extensions)
145+
err = s.walkDir(ctx, scanPath, false, sink, resolverSink, extensions)
147146
if err != nil {
148147
return errors.Wrap(err, "failed to walk directory")
149148
}

0 commit comments

Comments
 (0)