1- use { Token , SyntaxKind } ;
1+ use { SyntaxKind , Token } ;
22use syntax_kinds:: * ;
33
44mod ptr;
@@ -11,10 +11,11 @@ mod numbers;
1111use self :: numbers:: scan_number;
1212
1313mod strings;
14- use self :: strings:: { is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string} ;
14+ use self :: strings:: { is_string_literal_start, scan_byte_char_or_string, scan_char, scan_raw_string,
15+ scan_string} ;
1516
1617mod comments;
17- use self :: comments:: { scan_shebang , scan_comment } ;
18+ use self :: comments:: { scan_comment , scan_shebang } ;
1819
1920pub fn tokenize ( text : & str ) -> Vec < Token > {
2021 let mut text = text;
@@ -45,10 +46,10 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
4546 match c {
4647 '#' => if scan_shebang ( ptr) {
4748 return SHEBANG ;
48- }
49+ } ,
4950 '/' => if let Some ( kind) = scan_comment ( ptr) {
5051 return kind;
51- }
52+ } ,
5253 _ => ( ) ,
5354 }
5455
@@ -89,79 +90,91 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
8990 '%' => return PERCENT ,
9091
9192 // Multi-byte tokens.
92- '.' => return match ( ptr. next ( ) , ptr. nnext ( ) ) {
93- ( Some ( '.' ) , Some ( '.' ) ) => {
94- ptr. bump ( ) ;
95- ptr. bump ( ) ;
96- DOTDOTDOT
97- } ,
98- ( Some ( '.' ) , Some ( '=' ) ) => {
99- ptr. bump ( ) ;
100- ptr. bump ( ) ;
101- DOTDOTEQ
102- } ,
103- ( Some ( '.' ) , _) => {
104- ptr. bump ( ) ;
105- DOTDOT
106- } ,
107- _ => DOT
108- } ,
109- ':' => return match ptr. next ( ) {
110- Some ( ':' ) => {
111- ptr. bump ( ) ;
112- COLONCOLON
93+ '.' => {
94+ return match ( ptr. next ( ) , ptr. nnext ( ) ) {
95+ ( Some ( '.' ) , Some ( '.' ) ) => {
96+ ptr. bump ( ) ;
97+ ptr. bump ( ) ;
98+ DOTDOTDOT
99+ }
100+ ( Some ( '.' ) , Some ( '=' ) ) => {
101+ ptr. bump ( ) ;
102+ ptr. bump ( ) ;
103+ DOTDOTEQ
104+ }
105+ ( Some ( '.' ) , _) => {
106+ ptr. bump ( ) ;
107+ DOTDOT
108+ }
109+ _ => DOT ,
113110 }
114- _ => COLON
115- } ,
116- '=' => return match ptr. next ( ) {
117- Some ( '=' ) => {
118- ptr. bump ( ) ;
119- EQEQ
111+ }
112+ ':' => {
113+ return match ptr. next ( ) {
114+ Some ( ':' ) => {
115+ ptr. bump ( ) ;
116+ COLONCOLON
117+ }
118+ _ => COLON ,
120119 }
121- Some ( '>' ) => {
122- ptr. bump ( ) ;
123- FAT_ARROW
120+ }
121+ '=' => {
122+ return match ptr. next ( ) {
123+ Some ( '=' ) => {
124+ ptr. bump ( ) ;
125+ EQEQ
126+ }
127+ Some ( '>' ) => {
128+ ptr. bump ( ) ;
129+ FAT_ARROW
130+ }
131+ _ => EQ ,
124132 }
125- _ => EQ ,
126- } ,
127- '!' => return match ptr. next ( ) {
128- Some ( '=' ) => {
133+ }
134+ '!' => {
135+ return match ptr. next ( ) {
136+ Some ( '=' ) => {
137+ ptr. bump ( ) ;
138+ NEQ
139+ }
140+ _ => EXCL ,
141+ }
142+ }
143+ '-' => {
144+ return if ptr. next_is ( '>' ) {
129145 ptr. bump ( ) ;
130- NEQ
146+ THIN_ARROW
147+ } else {
148+ MINUS
131149 }
132- _ => EXCL ,
133- } ,
134- '-' => return if ptr. next_is ( '>' ) {
135- ptr. bump ( ) ;
136- THIN_ARROW
137- } else {
138- MINUS
139- } ,
150+ }
140151
141152 // If the character is an ident start not followed by another single
142153 // quote, then this is a lifetime name:
143- '\'' => return if ptr. next_is_p ( is_ident_start) && !ptr. nnext_is ( '\'' ) {
144- ptr. bump ( ) ;
145- while ptr. next_is_p ( is_ident_continue) {
146- ptr. bump ( ) ;
147- }
148- // lifetimes shouldn't end with a single quote
149- // if we find one, then this is an invalid character literal
150- if ptr. next_is ( '\'' ) {
154+ '\'' => {
155+ return if ptr. next_is_p ( is_ident_start) && !ptr. nnext_is ( '\'' ) {
151156 ptr. bump ( ) ;
152- return CHAR ; // TODO: error reporting
153- }
154- LIFETIME
155- } else {
156- scan_char ( ptr) ;
157- scan_literal_suffix ( ptr) ;
158- CHAR
159- } ,
157+ while ptr. next_is_p ( is_ident_continue) {
158+ ptr. bump ( ) ;
159+ }
160+ // lifetimes shouldn't end with a single quote
161+ // if we find one, then this is an invalid character literal
162+ if ptr. next_is ( '\'' ) {
163+ ptr. bump ( ) ;
164+ return CHAR ; // TODO: error reporting
165+ }
166+ LIFETIME
167+ } else {
168+ scan_char ( ptr) ;
169+ scan_literal_suffix ( ptr) ;
170+ CHAR
171+ } ;
172+ }
160173 'b' => {
161174 let kind = scan_byte_char_or_string ( ptr) ;
162175 scan_literal_suffix ( ptr) ;
163- return kind
164- } ,
176+ return kind;
177+ }
165178 '"' => {
166179 scan_string ( ptr) ;
167180 scan_literal_suffix ( ptr) ;
0 commit comments