Skip to content

Commit 543240f

Browse files
gabrittogabritto
authored
Create tokens with the right data (#2308)
1 parent cd451c4 commit 543240f

File tree

7 files changed

+94
-17
lines changed

7 files changed

+94
-17
lines changed

‎internal/ast/ast.go‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10781,6 +10781,7 @@ type SourceFile struct {
1078110781

1078210782
tokenCacheMu sync.Mutex
1078310783
tokenCache map[core.TextRange]*Node
10784+
tokenFactory *NodeFactory
1078410785
declarationMapMu sync.Mutex
1078510786
declarationMap map[string][]*Node
1078610787
}
@@ -10942,6 +10943,7 @@ func (node *SourceFile) GetOrCreateToken(
1094210943
pos int,
1094310944
end int,
1094410945
parent *Node,
10946+
flags TokenFlags,
1094510947
) *TokenNode {
1094610948
node.tokenCacheMu.Lock()
1094710949
defer node.tokenCacheMu.Unlock()
@@ -10959,13 +10961,57 @@ func (node *SourceFile) GetOrCreateToken(
1095910961
return token
1096010962
}
1096110963

10962-
token := newNode(kind, &Token{}, NodeFactoryHooks{})
10964+
token := createToken(kind, node, pos, end, flags)
1096310965
token.Loc = loc
1096410966
token.Parent = parent
1096510967
node.tokenCache[loc] = token
1096610968
return token
1096710969
}
1096810970

10971+
// `kind` should be a token kind.
10972+
func createToken(kind Kind, file *SourceFile, pos, end int, flags TokenFlags) *Node {
10973+
if file.tokenFactory == nil {
10974+
file.tokenFactory = NewNodeFactory(NodeFactoryHooks{})
10975+
}
10976+
text := file.text[pos:end]
10977+
switch kind {
10978+
case KindNumericLiteral:
10979+
literal := file.tokenFactory.NewNumericLiteral(text)
10980+
literal.AsNumericLiteral().TokenFlags = flags & TokenFlagsNumericLiteralFlags
10981+
return literal
10982+
case KindBigIntLiteral:
10983+
literal := file.tokenFactory.NewBigIntLiteral(text)
10984+
literal.AsBigIntLiteral().TokenFlags = flags & TokenFlagsNumericLiteralFlags
10985+
return literal
10986+
case KindStringLiteral:
10987+
literal := file.tokenFactory.NewStringLiteral(text)
10988+
literal.AsStringLiteral().TokenFlags = flags & TokenFlagsStringLiteralFlags
10989+
return literal
10990+
case KindJsxText, KindJsxTextAllWhiteSpaces:
10991+
return file.tokenFactory.NewJsxText(text, kind == KindJsxTextAllWhiteSpaces)
10992+
case KindRegularExpressionLiteral:
10993+
literal := file.tokenFactory.NewRegularExpressionLiteral(text)
10994+
literal.AsRegularExpressionLiteral().TokenFlags = flags & TokenFlagsRegularExpressionLiteralFlags
10995+
return literal
10996+
case KindNoSubstitutionTemplateLiteral:
10997+
literal := file.tokenFactory.NewNoSubstitutionTemplateLiteral(text)
10998+
literal.AsNoSubstitutionTemplateLiteral().TokenFlags = flags & TokenFlagsTemplateLiteralLikeFlags
10999+
return literal
11000+
case KindTemplateHead:
11001+
return file.tokenFactory.NewTemplateHead(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags)
11002+
case KindTemplateMiddle:
11003+
return file.tokenFactory.NewTemplateMiddle(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags)
11004+
case KindTemplateTail:
11005+
return file.tokenFactory.NewTemplateTail(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags)
11006+
case KindIdentifier:
11007+
return file.tokenFactory.NewIdentifier(text)
11008+
case KindPrivateIdentifier:
11009+
return file.tokenFactory.NewPrivateIdentifier(text)
11010+
default: // Punctuation and keywords
11011+
return file.tokenFactory.NewToken(kind)
11012+
}
11013+
}
11014+
1096911015
func IsSourceFile(node *Node) bool {
1097011016
return node.Kind == KindSourceFile
1097111017
}

‎internal/astnav/tokens.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -187,17 +187,18 @@ func getTokenAtPosition(
187187
tokenFullStart := scanner.TokenFullStart()
188188
tokenStart := core.IfElse(allowPositionInLeadingTrivia, tokenFullStart, scanner.TokenStart())
189189
tokenEnd := scanner.TokenEnd()
190+
flags := scanner.TokenFlags()
190191
if tokenStart <= position && (position < tokenEnd) {
191192
if token == ast.KindIdentifier || !ast.IsTokenKind(token) {
192193
if ast.IsJSDocKind(current.Kind) {
193194
return current
194195
}
195196
panic(fmt.Sprintf("did not expect %s to have %s in its trivia", current.Kind.String(), token.String()))
196197
}
197-
return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current)
198+
return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags)
198199
}
199200
if includePrecedingTokenAtEndPosition != nil && tokenEnd == position {
200-
prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current)
201+
prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags)
201202
if includePrecedingTokenAtEndPosition(prevToken) {
202203
return prevToken
203204
}
@@ -514,7 +515,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN
514515
tokenFullStart := scanner.TokenFullStart()
515516
tokenEnd := scanner.TokenEnd()
516517
startPos = tokenEnd
517-
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n))
518+
flags := scanner.TokenFlags()
519+
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags))
518520
scanner.Scan()
519521
}
520522
startPos = visitedNode.End()
@@ -531,7 +533,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN
531533
tokenFullStart := scanner.TokenFullStart()
532534
tokenEnd := scanner.TokenEnd()
533535
startPos = tokenEnd
534-
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n))
536+
flags := scanner.TokenFlags()
537+
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags))
535538
scanner.Scan()
536539
}
537540

@@ -616,8 +619,9 @@ func FindNextToken(previousToken *ast.Node, parent *ast.Node, file *ast.SourceFi
616619
tokenFullStart := scanner.TokenFullStart()
617620
tokenStart := scanner.TokenStart()
618621
tokenEnd := scanner.TokenEnd()
622+
flags := scanner.TokenFlags()
619623
if tokenStart == previousToken.End() {
620-
return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n)
624+
return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags)
621625
}
622626
panic(fmt.Sprintf("Expected to find next token at %d, got token %s at %d", previousToken.End(), token, tokenStart))
623627
}
@@ -690,7 +694,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So
690694
tokenKind := scan.Token()
691695
tokenFullStart := scan.TokenFullStart()
692696
tokenEnd := scan.TokenEnd()
693-
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode)
697+
flags := scan.TokenFlags()
698+
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
694699
if tokenKind == kind {
695700
foundChild = token
696701
return true
@@ -720,7 +725,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So
720725
tokenKind := scan.Token()
721726
tokenFullStart := scan.TokenFullStart()
722727
tokenEnd := scan.TokenEnd()
723-
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode)
728+
flags := scan.TokenFlags()
729+
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
724730
if tokenKind == kind {
725731
return token
726732
}

‎internal/fourslash/tests/completionsUnterminatedLiteral_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package fourslash_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/microsoft/typescript-go/internal/fourslash"
7+
. "github.com/microsoft/typescript-go/internal/fourslash/tests/util"
8+
"github.com/microsoft/typescript-go/internal/testutil"
9+
)
10+
11+
func TestCompletionsUnterminatedLiteral(t *testing.T) {
12+
t.Parallel()
13+
14+
defer testutil.RecoverAndFail(t, "Panic on fourslash test")
15+
const content = `// @noLib: true
16+
function foo(a"/*1*/`
17+
f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content)
18+
defer done()
19+
f.VerifyCompletions(t, "1", &fourslash.CompletionsExpectedList{
20+
ItemDefaults: &fourslash.CompletionsExpectedItemDefaults{
21+
CommitCharacters: &DefaultCommitCharacters,
22+
},
23+
Items: &fourslash.CompletionsExpectedItems{},
24+
})
25+
}

‎internal/ls/change/tracker.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func (t *Tracker) InsertNodeBefore(sourceFile *ast.SourceFile, before *ast.Node,
178178
// InsertModifierBefore inserts a modifier token (like 'type') before a node with a trailing space.
179179
func (t *Tracker) InsertModifierBefore(sourceFile *ast.SourceFile, modifier ast.Kind, before *ast.Node) {
180180
pos := astnav.GetStartOfNode(before, sourceFile, false)
181-
token := sourceFile.GetOrCreateToken(modifier, pos, pos, before.Parent)
181+
token := sourceFile.GetOrCreateToken(modifier, pos, pos, before.Parent, ast.TokenFlagsNone)
182182
t.InsertNodeAt(sourceFile, core.TextPos(pos), token, NodeOptions{Suffix: " "})
183183
}
184184

@@ -262,7 +262,7 @@ func (t *Tracker) endPosForInsertNodeAfter(sourceFile *ast.SourceFile, after *as
262262
endPos := t.converters.PositionToLineAndCharacter(sourceFile, core.TextPos(after.End()))
263263
t.ReplaceRange(sourceFile,
264264
lsproto.Range{Start: endPos, End: endPos},
265-
sourceFile.GetOrCreateToken(ast.KindSemicolonToken, after.End(), after.End(), after.Parent),
265+
sourceFile.GetOrCreateToken(ast.KindSemicolonToken, after.End(), after.End(), after.Parent, ast.TokenFlagsNone),
266266
NodeOptions{},
267267
)
268268
}
@@ -347,7 +347,7 @@ func (t *Tracker) InsertNodeInListAfter(sourceFile *ast.SourceFile, after *ast.N
347347

348348
// insert separator immediately following the 'after' node to preserve comments in trailing trivia
349349
// !!! formatcontext
350-
t.ReplaceRange(sourceFile, lsproto.Range{Start: end, End: end}, sourceFile.GetOrCreateToken(separator, after.End(), after.End()+len(separatorString), after.Parent), NodeOptions{})
350+
t.ReplaceRange(sourceFile, lsproto.Range{Start: end, End: end}, sourceFile.GetOrCreateToken(separator, after.End(), after.End()+len(separatorString), after.Parent, ast.TokenFlagsNone), NodeOptions{})
351351
// use the same indentation as 'after' item
352352
indentation := format.FindFirstNonWhitespaceColumn(afterStartLinePosition, afterStart, sourceFile, t.formatSettings)
353353
// insert element before the line break on the line that contains 'after' element

‎internal/ls/lsutil/children.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ func GetLastChild(node *ast.Node, sourceFile *ast.SourceFile) *ast.Node {
2525
tokenKind := scanner.Token()
2626
tokenFullStart := scanner.TokenFullStart()
2727
tokenEnd := scanner.TokenEnd()
28-
lastToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node)
28+
lastToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node, scanner.TokenFlags())
2929
startPos = tokenEnd
3030
scanner.Scan()
3131
}
@@ -108,7 +108,7 @@ func GetFirstToken(node *ast.Node, sourceFile *ast.SourceFile) *ast.Node {
108108
tokenKind := scanner.Token()
109109
tokenFullStart := scanner.TokenFullStart()
110110
tokenEnd := scanner.TokenEnd()
111-
firstToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node)
111+
firstToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node, scanner.TokenFlags())
112112
}
113113

114114
if firstToken != nil {

‎internal/ls/signaturehelp.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1241,7 +1241,7 @@ func getTokenFromNodeList(nodeList *ast.NodeList, nodeListParent *ast.Node, sour
12411241
token := scanner.Token()
12421242
tokenFullStart := scanner.TokenFullStart()
12431243
tokenEnd := scanner.TokenEnd()
1244-
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, nodeListParent))
1244+
tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, nodeListParent, scanner.TokenFlags()))
12451245
left = tokenEnd
12461246
}
12471247
}

‎internal/ls/utilities.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ func nodeEndsWith(n *ast.Node, expectedLastToken ast.Kind, sourceFile *ast.Sourc
752752
tokenKind := scanner.Token()
753753
tokenFullStart := scanner.TokenFullStart()
754754
tokenEnd := scanner.TokenEnd()
755-
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, n)
755+
token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, n, scanner.TokenFlags())
756756
lastNodeAndTokens = append(lastNodeAndTokens, token)
757757
startPos = tokenEnd
758758
scanner.Scan()
@@ -1571,7 +1571,7 @@ func getChildrenFromNonJSDocNode(node *ast.Node, sourceFile *ast.SourceFile) []*
15711571
token := scanner.Token()
15721572
tokenFullStart := scanner.TokenFullStart()
15731573
tokenEnd := scanner.TokenEnd()
1574-
children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node))
1574+
children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node, scanner.TokenFlags()))
15751575
pos = tokenEnd
15761576
scanner.Scan()
15771577
}
@@ -1583,7 +1583,7 @@ func getChildrenFromNonJSDocNode(node *ast.Node, sourceFile *ast.SourceFile) []*
15831583
token := scanner.Token()
15841584
tokenFullStart := scanner.TokenFullStart()
15851585
tokenEnd := scanner.TokenEnd()
1586-
children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node))
1586+
children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node, scanner.TokenFlags()))
15871587
pos = tokenEnd
15881588
scanner.Scan()
15891589
}

0 commit comments

Comments
 (0)