Skip to content

Commit 82f5d8c

Browse files
hailin0hailin0
authored
[Improve][SQL-Transform] Remove escape identifier from output fields (apache#7297)
1 parent 34a6b8e commit 82f5d8c

File tree

5 files changed

+191
-6
lines changed

5 files changed

+191
-6
lines changed

‎seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ transform {
4949
Sql {
5050
source_table_name = "fake"
5151
result_table_name = "fake1"
52-
query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(c1 as double) as c1_1, cast(c1 as DECIMAL(10,2)) as c1_2, cast(c2 as DATE) as c2_1, coalesce(c3,'Unknown') c3_1, ifnull(c3,'Unknown') c3_2, ifnull(nullif(name,'Joy Ding'),'NULL') name1, nullif(name,'Joy Ding_') name2, cast(c4 as timestamp) as c4_1, cast(c4 as decimal(17,4)) as c4_2, cast(c5 as date) as c5, cast(c6 as time) as c6, cast(name as bytes) as c7 from fake"
52+
query = "select cast(id as STRING) as id, cast(id as INT) as id2, cast(id as DOUBLE) as id3 , cast(c1 as double) as c1_1, cast(c1 as DECIMAL(10,2)) as c1_2, cast(c2 as DATE) as c2_1, coalesce(c3,'Unknown') c3_1, ifnull(c3,'Unknown') c3_2, ifnull(nullif(name,'Joy Ding'),'NULL') name1, nullif(name,'Joy Ding_') name2, cast(c4 as timestamp) as c4_1, cast(c4 as decimal(17,4)) as c4_2, cast(c5 as date) as c5, cast(c6 as time) as c6, cast(name as bytes) as c7, name as `apply` from fake"
5353
}
5454
}
5555

@@ -164,6 +164,13 @@ sink {
164164
rule_type = NOT_NULL
165165
}
166166
]
167+
},
168+
{
169+
field_name = "apply"
170+
field_type = "string"
171+
field_value = [
172+
{equals_to = "Joy Ding"}
173+
]
167174
}
168175
]
169176
}

‎seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngine.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@
5050

5151
public class ZetaSQLEngine implements SQLEngine {
5252
private static final Logger log = LoggerFactory.getLogger(ZetaSQLEngine.class);
53+
public static final String ESCAPE_IDENTIFIER = "`";
54+
5355
private String inputTableName;
5456
@Nullable private String catalogTableName;
5557
private SeaTunnelRowType inputRowType;
@@ -193,9 +195,13 @@ public SeaTunnelRowType typeMapping(List<String> inputColumnsMapping) {
193195
} else if (selectItem instanceof SelectExpressionItem) {
194196
SelectExpressionItem expressionItem = (SelectExpressionItem) selectItem;
195197
Expression expression = expressionItem.getExpression();
196-
197198
if (expressionItem.getAlias() != null) {
198-
fieldNames[idx] = expressionItem.getAlias().getName();
199+
String aliasName = expressionItem.getAlias().getName();
200+
if (aliasName.startsWith(ESCAPE_IDENTIFIER)
201+
&& aliasName.endsWith(ESCAPE_IDENTIFIER)) {
202+
aliasName = aliasName.substring(1, aliasName.length() - 1);
203+
}
204+
fieldNames[idx] = aliasName;
199205
} else {
200206
if (expression instanceof Column) {
201207
fieldNames[idx] = ((Column) expression).getColumnName();

‎seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,13 @@ public Object computeForValue(Expression expression, Object[] inputFields) {
227227
Column columnExp = (Column) expression;
228228
String columnName = columnExp.getColumnName();
229229
int index = inputRowType.indexOf(columnName, false);
230+
if (index == -1
231+
&& columnName.startsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)
232+
&& columnName.endsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)) {
233+
columnName = columnName.substring(1, columnName.length() - 1);
234+
index = inputRowType.indexOf(columnName, false);
235+
}
236+
230237
if (index != -1) {
231238
return inputFields[index];
232239
} else {
@@ -237,11 +244,26 @@ public Object computeForValue(Expression expression, Object[] inputFields) {
237244
SeaTunnelRow parRowValues = new SeaTunnelRow(inputFields);
238245
Object res = parRowValues;
239246
for (int i = 0; i < deep; i++) {
247+
String key = columnNames[i];
240248
if (parDataType instanceof MapType) {
241-
return ((Map) res).get(columnNames[i]);
249+
Map<String, Object> mapValue = ((Map) res);
250+
if (mapValue.containsKey(key)) {
251+
return mapValue.get(key);
252+
} else if (key.startsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)
253+
&& key.endsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)) {
254+
key = key.substring(1, key.length() - 1);
255+
return mapValue.get(key);
256+
}
257+
return null;
242258
}
243259
parRowValues = (SeaTunnelRow) res;
244-
int idx = ((SeaTunnelRowType) parDataType).indexOf(columnNames[i], false);
260+
int idx = ((SeaTunnelRowType) parDataType).indexOf(key, false);
261+
if (idx == -1
262+
&& key.startsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)
263+
&& key.endsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)) {
264+
key = key.substring(1, key.length() - 1);
265+
idx = ((SeaTunnelRowType) parDataType).indexOf(key, false);
266+
}
245267
if (idx == -1) {
246268
throw new IllegalArgumentException(
247269
String.format("can't find field [%s]", fullyQualifiedName));

‎seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,13 @@ public SeaTunnelDataType<?> getExpressionType(Expression expression) {
111111
Column columnExp = (Column) expression;
112112
String columnName = columnExp.getColumnName();
113113
int index = inputRowType.indexOf(columnName, false);
114+
if (index == -1
115+
&& columnName.startsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)
116+
&& columnName.endsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)) {
117+
columnName = columnName.substring(1, columnName.length() - 1);
118+
index = inputRowType.indexOf(columnName, false);
119+
}
120+
114121
if (index != -1) {
115122
return inputRowType.getFieldType(index);
116123
} else {
@@ -121,7 +128,14 @@ public SeaTunnelDataType<?> getExpressionType(Expression expression) {
121128
SeaTunnelRowType parRowType = inputRowType;
122129
SeaTunnelDataType<?> filedTypeRes = null;
123130
for (int i = 0; i < deep; i++) {
124-
int idx = parRowType.indexOf(columnNames[i], false);
131+
String key = columnNames[i];
132+
int idx = parRowType.indexOf(key, false);
133+
if (idx == -1
134+
&& key.startsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)
135+
&& key.endsWith(ZetaSQLEngine.ESCAPE_IDENTIFIER)) {
136+
key = key.substring(1, key.length() - 1);
137+
idx = parRowType.indexOf(key, false);
138+
}
125139
if (idx == -1) {
126140
throw new IllegalArgumentException(
127141
String.format("can't find field [%s]", fullyQualifiedName));

‎seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformTest.java‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,22 @@
1919

2020
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
2121
import org.apache.seatunnel.api.table.catalog.CatalogTable;
22+
import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
2223
import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
2324
import org.apache.seatunnel.api.table.catalog.TableIdentifier;
2425
import org.apache.seatunnel.api.table.catalog.TableSchema;
2526
import org.apache.seatunnel.api.table.type.BasicType;
2627
import org.apache.seatunnel.api.table.type.LocalTimeType;
28+
import org.apache.seatunnel.api.table.type.MapType;
2729
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
30+
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
2831
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
2932

3033
import org.junit.jupiter.api.Assertions;
3134
import org.junit.jupiter.api.Test;
3235

3336
import java.util.ArrayList;
37+
import java.util.Collections;
3438
import java.util.HashMap;
3539
import java.util.Objects;
3640

@@ -144,4 +148,136 @@ private CatalogTable getCatalogTable() {
144148
new ArrayList<>(),
145149
"It has column information.");
146150
}
151+
152+
@Test
153+
public void testEscapeIdentifier() {
154+
String tableName = "test";
155+
String[] fields = new String[] {"id", "apply"};
156+
CatalogTable table =
157+
CatalogTableUtil.getCatalogTable(
158+
tableName,
159+
new SeaTunnelRowType(
160+
fields,
161+
new SeaTunnelDataType[] {
162+
BasicType.INT_TYPE, BasicType.STRING_TYPE
163+
}));
164+
ReadonlyConfig config =
165+
ReadonlyConfig.fromMap(
166+
Collections.singletonMap(
167+
"query",
168+
"select id, trim(`apply`) as `apply` from test where `apply` = 'a'"));
169+
SQLTransform sqlTransform = new SQLTransform(config, table);
170+
TableSchema tableSchema = sqlTransform.transformTableSchema();
171+
SeaTunnelRow result =
172+
sqlTransform.transformRow(
173+
new SeaTunnelRow(new Object[] {Integer.valueOf(1), String.valueOf("a")}));
174+
Assertions.assertEquals("apply", tableSchema.getFieldNames()[1]);
175+
Assertions.assertEquals("a", result.getField(1));
176+
result =
177+
sqlTransform.transformRow(
178+
new SeaTunnelRow(new Object[] {Integer.valueOf(1), String.valueOf("b")}));
179+
Assertions.assertNull(result);
180+
181+
config =
182+
ReadonlyConfig.fromMap(
183+
Collections.singletonMap(
184+
"query",
185+
"select id, IFNULL(`apply`, '1') as `apply` from test where `apply` = 'a'"));
186+
sqlTransform = new SQLTransform(config, table);
187+
tableSchema = sqlTransform.transformTableSchema();
188+
result =
189+
sqlTransform.transformRow(
190+
new SeaTunnelRow(new Object[] {Integer.valueOf(1), String.valueOf("a")}));
191+
Assertions.assertEquals("apply", tableSchema.getFieldNames()[1]);
192+
Assertions.assertEquals(
193+
BasicType.STRING_TYPE, tableSchema.getColumns().get(1).getDataType());
194+
Assertions.assertEquals("a", result.getField(1));
195+
196+
table =
197+
CatalogTableUtil.getCatalogTable(
198+
tableName,
199+
new SeaTunnelRowType(
200+
fields,
201+
new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.LONG_TYPE}));
202+
config =
203+
ReadonlyConfig.fromMap(
204+
Collections.singletonMap(
205+
"query",
206+
"select id, `apply` + 1 as `apply` from test where `apply` > 0"));
207+
sqlTransform = new SQLTransform(config, table);
208+
tableSchema = sqlTransform.transformTableSchema();
209+
result =
210+
sqlTransform.transformRow(
211+
new SeaTunnelRow(new Object[] {Integer.valueOf(1), Long.valueOf(1)}));
212+
Assertions.assertEquals("apply", tableSchema.getFieldNames()[1]);
213+
Assertions.assertEquals(BasicType.LONG_TYPE, tableSchema.getColumns().get(1).getDataType());
214+
Assertions.assertEquals(Long.valueOf(2), result.getField(1));
215+
result =
216+
sqlTransform.transformRow(
217+
new SeaTunnelRow(new Object[] {Integer.valueOf(1), Long.valueOf(0)}));
218+
Assertions.assertNull(result);
219+
220+
table =
221+
CatalogTableUtil.getCatalogTable(
222+
tableName,
223+
new SeaTunnelRowType(
224+
fields,
225+
new SeaTunnelDataType[] {
226+
BasicType.INT_TYPE,
227+
new MapType<String, String>(
228+
BasicType.STRING_TYPE, BasicType.STRING_TYPE)
229+
}));
230+
config =
231+
ReadonlyConfig.fromMap(
232+
Collections.singletonMap(
233+
"query",
234+
"select id, `apply`.k1 as `apply` from test where `apply`.k1 = 'a'"));
235+
sqlTransform = new SQLTransform(config, table);
236+
tableSchema = sqlTransform.transformTableSchema();
237+
result =
238+
sqlTransform.transformRow(
239+
new SeaTunnelRow(
240+
new Object[] {
241+
Integer.valueOf(1), Collections.singletonMap("k1", "a")
242+
}));
243+
Assertions.assertEquals("apply", tableSchema.getFieldNames()[1]);
244+
Assertions.assertEquals(
245+
BasicType.STRING_TYPE, tableSchema.getColumns().get(1).getDataType());
246+
Assertions.assertEquals("a", result.getField(1));
247+
result =
248+
sqlTransform.transformRow(
249+
new SeaTunnelRow(
250+
new Object[] {
251+
Integer.valueOf(1), Collections.singletonMap("k1", "b")
252+
}));
253+
Assertions.assertNull(result);
254+
255+
table =
256+
CatalogTableUtil.getCatalogTable(
257+
tableName,
258+
new SeaTunnelRowType(
259+
new String[] {"id", "map"},
260+
new SeaTunnelDataType[] {
261+
BasicType.INT_TYPE,
262+
new MapType<String, String>(
263+
BasicType.STRING_TYPE, BasicType.STRING_TYPE)
264+
}));
265+
config =
266+
ReadonlyConfig.fromMap(
267+
Collections.singletonMap(
268+
"query",
269+
"select id, map.`apply` as `apply` from test where map.`apply` = 'a'"));
270+
sqlTransform = new SQLTransform(config, table);
271+
tableSchema = sqlTransform.transformTableSchema();
272+
result =
273+
sqlTransform.transformRow(
274+
new SeaTunnelRow(
275+
new Object[] {
276+
Integer.valueOf(1), Collections.singletonMap("apply", "a")
277+
}));
278+
Assertions.assertEquals("apply", tableSchema.getFieldNames()[1]);
279+
Assertions.assertEquals(
280+
BasicType.STRING_TYPE, tableSchema.getColumns().get(1).getDataType());
281+
Assertions.assertEquals("a", result.getField(1));
282+
}
147283
}

0 commit comments

Comments
 (0)