Skip to content

Commit 80321bc

Browse files
dogancanbakirdogancanbakir
committed
add variable support to extractors
1 parent 2997735 commit 80321bc

File tree

10 files changed

+64
-19
lines changed

10 files changed

+64
-19
lines changed

‎pkg/operators/extractors/compile.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/itchyny/gojq"
1010
"github.com/projectdiscovery/nuclei/v3/pkg/operators/cache"
1111
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
12+
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
1213
)
1314

1415
// CompileExtractors performs the initial setup operation on an extractor
@@ -21,6 +22,11 @@ func (e *Extractor) CompileExtractors() error {
2122
e.extractorType = computedType
2223
// Compile the regexes
2324
for _, regex := range e.Regex {
25+
if varErr := expressions.ContainsUnresolvedVariables(regex); varErr != nil {
26+
e.regexCompiled = append(e.regexCompiled, nil)
27+
continue
28+
}
29+
2430
if cached, err := cache.Regex().GetIFPresent(regex); err == nil && cached != nil {
2531
e.regexCompiled = append(e.regexCompiled, cached)
2632
continue
@@ -37,6 +43,10 @@ func (e *Extractor) CompileExtractors() error {
3743
}
3844

3945
for _, query := range e.JSON {
46+
if varErr := expressions.ContainsUnresolvedVariables(query); varErr != nil {
47+
e.jsonCompiled = append(e.jsonCompiled, nil)
48+
continue
49+
}
4050
query, err := gojq.Parse(query)
4151
if err != nil {
4252
return fmt.Errorf("could not parse json: %s", query)

‎pkg/operators/extractors/extract.go‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,38 @@ package extractors
22

33
import (
44
"fmt"
5+
"regexp"
56
"strings"
67

78
"github.com/antchfx/htmlquery"
89
"github.com/antchfx/xmlquery"
10+
"github.com/itchyny/gojq"
911

12+
"github.com/projectdiscovery/gologger"
13+
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
1014
"github.com/projectdiscovery/nuclei/v3/pkg/types"
1115
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
1216
)
1317

1418
// ExtractRegex extracts text from a corpus and returns it
15-
func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
19+
func (e *Extractor) ExtractRegex(corpus string, data map[string]interface{}) map[string]struct{} {
1620
results := make(map[string]struct{})
1721

1822
groupPlusOne := e.RegexGroup + 1
19-
for _, regex := range e.regexCompiled {
23+
for i, regex := range e.regexCompiled {
24+
if varErr := expressions.ContainsUnresolvedVariables(e.Regex[i]); varErr != nil {
25+
regexStr, err := expressions.Evaluate(e.Regex[i], data)
26+
if err != nil {
27+
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.Regex[i], err.Error())
28+
continue
29+
}
30+
regex, err = regexp.Compile(regexStr)
31+
if err != nil {
32+
gologger.Warning().Msgf("Could not compile regex: %s, error: %s", regexStr, err.Error())
33+
continue
34+
}
35+
}
36+
2037
// skip prefix short-circuit for case-insensitive patterns
2138
rstr := regex.String()
2239
if !strings.Contains(rstr, "(?i") {
@@ -138,7 +155,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
138155
}
139156

140157
// ExtractJSON extracts text from a corpus using JQ queries and returns it
141-
func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
158+
func (e *Extractor) ExtractJSON(corpus string, data map[string]interface{}) map[string]struct{} {
142159
results := make(map[string]struct{})
143160

144161
var jsonObj interface{}
@@ -147,7 +164,25 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
147164
return results
148165
}
149166

150-
for _, k := range e.jsonCompiled {
167+
for i, k := range e.jsonCompiled {
168+
if varErr := expressions.ContainsUnresolvedVariables(e.JSON[i]); varErr != nil {
169+
jsonStr, err := expressions.Evaluate(e.JSON[i], data)
170+
if err != nil {
171+
gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.JSON[i], err.Error())
172+
continue
173+
}
174+
query, err := gojq.Parse(jsonStr)
175+
if err != nil {
176+
gologger.Warning().Msgf("Could not parse json: %s, error: %s", jsonStr, err.Error())
177+
continue
178+
}
179+
k, err = gojq.Compile(query)
180+
if err != nil {
181+
gologger.Warning().Msgf("Could not compile json: %s, error: %s", jsonStr, err.Error())
182+
continue
183+
}
184+
}
185+
151186
iter := k.Run(jsonObj)
152187
for {
153188
v, ok := iter.Next()

‎pkg/operators/extractors/extract_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ func TestExtractor_ExtractRegex(t *testing.T) {
1111
err := e.CompileExtractors()
1212
require.Nil(t, err)
1313

14-
got := e.ExtractRegex("RegEx")
14+
got := e.ExtractRegex("RegEx", map[string]interface{}{})
1515
require.Equal(t, map[string]struct{}{"RegEx": {}}, got)
1616

17-
got = e.ExtractRegex("regex")
17+
got = e.ExtractRegex("regex", map[string]interface{}{})
1818
require.Equal(t, map[string]struct{}{}, got)
1919
}
2020

@@ -70,10 +70,10 @@ func TestExtractor_ExtractJSON(t *testing.T) {
7070
err := e.CompileExtractors()
7171
require.Nil(t, err)
7272

73-
got := e.ExtractJSON(`[{"id": 1}]`)
73+
got := e.ExtractJSON(`[{"id": 1}]`, map[string]interface{}{})
7474
require.Equal(t, map[string]struct{}{"1": {}}, got)
7575

76-
got = e.ExtractJSON(`{"id": 1}`)
76+
got = e.ExtractJSON(`{"id": 1}`, map[string]interface{}{})
7777
require.Equal(t, map[string]struct{}{}, got)
7878
}
7979

‎pkg/protocols/dns/operators.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
5757

5858
switch extractor.GetType() {
5959
case extractors.RegexExtractor:
60-
return extractor.ExtractRegex(types.ToString(item))
60+
return extractor.ExtractRegex(types.ToString(item), data)
6161
case extractors.KValExtractor:
6262
return extractor.ExtractKval(data)
6363
case extractors.DSLExtractor:

‎pkg/protocols/file/operators.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
4545

4646
switch extractor.GetType() {
4747
case extractors.RegexExtractor:
48-
return extractor.ExtractRegex(itemStr)
48+
return extractor.ExtractRegex(itemStr, data)
4949
case extractors.KValExtractor:
5050
return extractor.ExtractKval(data)
5151
case extractors.JSONExtractor:
52-
return extractor.ExtractJSON(itemStr)
52+
return extractor.ExtractJSON(itemStr, data)
5353
case extractors.XPathExtractor:
5454
return extractor.ExtractXPath(itemStr)
5555
case extractors.DSLExtractor:

‎pkg/protocols/headless/operators.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,15 +71,15 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
7171

7272
switch extractor.GetType() {
7373
case extractors.RegexExtractor:
74-
return extractor.ExtractRegex(itemStr)
74+
return extractor.ExtractRegex(itemStr, data)
7575
case extractors.KValExtractor:
7676
return extractor.ExtractKval(data)
7777
case extractors.DSLExtractor:
7878
return extractor.ExtractDSL(data)
7979
case extractors.XPathExtractor:
8080
return extractor.ExtractXPath(itemStr)
8181
case extractors.JSONExtractor:
82-
return extractor.ExtractJSON(itemStr)
82+
return extractor.ExtractJSON(itemStr, data)
8383
}
8484
return nil
8585
}

‎pkg/protocols/http/operators.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
6868
}
6969
switch extractor.GetType() {
7070
case extractors.RegexExtractor:
71-
return extractor.ExtractRegex(item)
71+
return extractor.ExtractRegex(item, data)
7272
case extractors.KValExtractor:
7373
return extractor.ExtractKval(data)
7474
case extractors.XPathExtractor:
7575
return extractor.ExtractXPath(item)
7676
case extractors.JSONExtractor:
77-
return extractor.ExtractJSON(item)
77+
return extractor.ExtractJSON(item, data)
7878
case extractors.DSLExtractor:
7979
return extractor.ExtractDSL(data)
8080
}

‎pkg/protocols/network/operators.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
4646

4747
switch extractor.GetType() {
4848
case extractors.RegexExtractor:
49-
return extractor.ExtractRegex(itemStr)
49+
return extractor.ExtractRegex(itemStr, data)
5050
case extractors.KValExtractor:
5151
return extractor.ExtractKval(data)
5252
case extractors.DSLExtractor:

‎pkg/protocols/offlinehttp/operators.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
6767
}
6868
switch extractor.GetType() {
6969
case extractors.RegexExtractor:
70-
return extractor.ExtractRegex(item)
70+
return extractor.ExtractRegex(item, data)
7171
case extractors.KValExtractor:
7272
return extractor.ExtractKval(data)
7373
case extractors.DSLExtractor:

‎pkg/protocols/protocols.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -388,11 +388,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E
388388

389389
switch extractor.GetType() {
390390
case extractors.RegexExtractor:
391-
return extractor.ExtractRegex(itemStr)
391+
return extractor.ExtractRegex(itemStr, data)
392392
case extractors.KValExtractor:
393393
return extractor.ExtractKval(data)
394394
case extractors.JSONExtractor:
395-
return extractor.ExtractJSON(itemStr)
395+
return extractor.ExtractJSON(itemStr, data)
396396
case extractors.XPathExtractor:
397397
return extractor.ExtractXPath(itemStr)
398398
case extractors.DSLExtractor:

0 commit comments

Comments
 (0)