Skip to content

Commit 6145da9

Browse files
hawk9821hawk9821
authored
[Improve][Zeta] Handle user privacy when submitting a task print config logs (#7247)
1 parent 1bba723 commit 6145da9

File tree

6 files changed

+156
-7
lines changed

6 files changed

+156
-7
lines changed

‎config/seatunnel.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ seatunnel:
3333
plugin-config:
3434
namespace: /tmp/seatunnel/checkpoint_snapshot
3535
storage.type: hdfs
36-
fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
36+
fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission

‎seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java‎

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,26 @@
2222
import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions;
2323
import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
2424
import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
25+
import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax;
2526
import org.apache.seatunnel.shade.com.typesafe.config.impl.Parseable;
2627

2728
import org.apache.seatunnel.api.configuration.ConfigAdapter;
29+
import org.apache.seatunnel.common.utils.JsonUtils;
2830
import org.apache.seatunnel.common.utils.ParserException;
2931

3032
import lombok.NonNull;
3133
import lombok.extern.slf4j.Slf4j;
3234

3335
import java.nio.file.Path;
3436
import java.nio.file.Paths;
37+
import java.util.Arrays;
3538
import java.util.List;
3639
import java.util.Map;
3740
import java.util.Objects;
3841
import java.util.Optional;
42+
import java.util.stream.Collectors;
43+
44+
import static org.apache.seatunnel.core.starter.utils.ConfigShadeUtils.DEFAULT_SENSITIVE_KEYWORDS;
3945

4046
/** Used to build the {@link Config} from config file. */
4147
@Slf4j
@@ -76,14 +82,19 @@ public static Config of(@NonNull Path filePath, List<String> variables) {
7682
adapterSupplier
7783
.map(adapter -> of(adapter, filePath, variables))
7884
.orElseGet(() -> ofInner(filePath, variables));
85+
boolean isJson = filePath.getFileName().toString().endsWith(".json");
86+
log.info(
87+
"Parsed config file: \n{}",
88+
mapToString(configDesensitization(config.root().unwrapped()), isJson));
7989
return config;
8090
}
8191

8292
public static Config of(@NonNull Map<String, Object> objectMap) {
83-
return of(objectMap, false);
93+
return of(objectMap, false, false);
8494
}
8595

86-
public static Config of(@NonNull Map<String, Object> objectMap, boolean isEncrypt) {
96+
public static Config of(
97+
@NonNull Map<String, Object> objectMap, boolean isEncrypt, boolean isJson) {
8798
log.info("Loading config file from objectMap");
8899
Config config =
89100
ConfigFactory.parseMap(objectMap)
@@ -94,9 +105,49 @@ public static Config of(@NonNull Map<String, Object> objectMap, boolean isEncryp
94105
if (!isEncrypt) {
95106
config = ConfigShadeUtils.decryptConfig(config);
96107
}
108+
log.info(
109+
"Parsed config file: \n{}",
110+
mapToString(configDesensitization(config.root().unwrapped()), isJson));
97111
return config;
98112
}
99113

114+
public static Map<String, Object> configDesensitization(Map<String, Object> configMap) {
115+
return configMap.entrySet().stream()
116+
.collect(
117+
Collectors.toMap(
118+
Map.Entry::getKey,
119+
entry -> {
120+
String key = entry.getKey();
121+
if (Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)
122+
.contains(key.toLowerCase())) {
123+
return "******";
124+
}
125+
Object value = entry.getValue();
126+
if (value instanceof Map) {
127+
if ("schema".equals(key)) {
128+
return value;
129+
}
130+
return configDesensitization((Map<String, Object>) value);
131+
} else if (value instanceof List) {
132+
return ((List<?>) value)
133+
.stream()
134+
.map(
135+
v -> {
136+
if (v instanceof Map) {
137+
return configDesensitization(
138+
(Map<
139+
String,
140+
Object>)
141+
v);
142+
}
143+
return v;
144+
})
145+
.collect(Collectors.toList());
146+
}
147+
return value;
148+
}));
149+
}
150+
100151
public static Config of(
101152
@NonNull ConfigAdapter configAdapter, @NonNull Path filePath, List<String> variables) {
102153
log.info("With config adapter spi {}", configAdapter.getClass().getName());
@@ -133,4 +184,18 @@ private static Config backfillUserVariables(Config config, List<String> variable
133184
}
134185
return config;
135186
}
187+
188+
public static String mapToString(Map<String, Object> configMap, boolean isJson) {
189+
ConfigRenderOptions configRenderOptions =
190+
ConfigRenderOptions.concise().setFormatted(true).setJson(isJson);
191+
ConfigParseOptions configParseOptions =
192+
ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON);
193+
Config config =
194+
ConfigFactory.parseString(JsonUtils.toJsonString(configMap), configParseOptions)
195+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
196+
.resolveWith(
197+
ConfigFactory.systemProperties(),
198+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
199+
return config.root().render(configRenderOptions);
200+
}
136201
}

‎seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ public final class ConfigShadeUtils {
4747

4848
private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier";
4949

50-
private static final String[] DEFAULT_SENSITIVE_OPTIONS =
51-
new String[] {"password", "username", "auth"};
50+
public static final String[] DEFAULT_SENSITIVE_KEYWORDS =
51+
new String[] {"password", "username", "auth", "token"};
5252

5353
private static final Map<String, ConfigShade> CONFIG_SHADES = new HashMap<>();
5454

@@ -126,7 +126,7 @@ public static Config encryptConfig(String identifier, Config config) {
126126
@SuppressWarnings("unchecked")
127127
private static Config processConfig(String identifier, Config config, boolean isDecrypted) {
128128
ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE);
129-
List<String> sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_OPTIONS));
129+
List<String> sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS));
130130
sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions()));
131131
BiFunction<String, Object, String> processFunction =
132132
(key, value) -> {

‎seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,17 @@
1919

2020
import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
2121
import org.apache.seatunnel.shade.com.typesafe.config.Config;
22+
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
2223
import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject;
24+
import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
2325

2426
import org.apache.seatunnel.api.configuration.ConfigShade;
2527
import org.apache.seatunnel.common.utils.JsonUtils;
2628

2729
import org.junit.jupiter.api.Assertions;
2830
import org.junit.jupiter.api.Test;
2931

32+
import com.beust.jcommander.internal.Lists;
3033
import lombok.extern.slf4j.Slf4j;
3134

3235
import java.net.URISyntaxException;
@@ -68,6 +71,46 @@ public void testParseConfig() throws URISyntaxException {
6871
config.getConfigList("source").get(0).getString("password"), PASSWORD);
6972
}
7073

74+
@Test
75+
public void testUsePrivacyHandlerHocon() throws URISyntaxException {
76+
URL resource = ConfigShadeTest.class.getResource("/config.shade.conf");
77+
Assertions.assertNotNull(resource);
78+
Config config = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList());
79+
config =
80+
ConfigFactory.parseMap(
81+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
82+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
83+
.resolveWith(
84+
ConfigFactory.systemProperties(),
85+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
86+
Assertions.assertEquals(
87+
config.getConfigList("source").get(0).getString("username"), "******");
88+
Assertions.assertEquals(
89+
config.getConfigList("source").get(0).getString("password"), "******");
90+
String conf = ConfigBuilder.mapToString(config.root().unwrapped(), false);
91+
Assertions.assertTrue(conf.contains("username=\"******\""));
92+
}
93+
94+
@Test
95+
public void testUsePrivacyHandlerJson() throws URISyntaxException {
96+
URL resource = ConfigShadeTest.class.getResource("/config.shade.json");
97+
Assertions.assertNotNull(resource);
98+
Config config = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList());
99+
config =
100+
ConfigFactory.parseMap(
101+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
102+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
103+
.resolveWith(
104+
ConfigFactory.systemProperties(),
105+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
106+
Assertions.assertEquals(
107+
config.getConfigList("source").get(0).getString("username"), "******");
108+
Assertions.assertEquals(
109+
config.getConfigList("source").get(0).getString("password"), "******");
110+
String json = ConfigBuilder.mapToString(config.root().unwrapped(), true);
111+
Assertions.assertTrue(json.contains("\"password\" : \"******\""));
112+
}
113+
71114
@Test
72115
public void testVariableReplacement() throws URISyntaxException {
73116
String jobName = "seatunnel variable test job";

‎seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"env" : {
3+
"shade.identifier" : "base64",
4+
"parallelism" : 1
5+
},
6+
"source" : [
7+
{
8+
"plugin_name" : "MySQL-CDC",
9+
"base-url" : "jdbc:mysql://localhost:56725",
10+
"username" : "c2VhdHVubmVs",
11+
"password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
12+
"hostname" : "127.0.0.1",
13+
"port" : 56725,
14+
"database-name" : "inventory_vwyw0n",
15+
"parallelism" : 1,
16+
"table-name" : "products",
17+
"server-id" : 5656,
18+
"schema" : {
19+
"fields" : {
20+
"name" : "string",
21+
"age" : "int",
22+
"sex" : "boolean"
23+
}
24+
},
25+
"result_table_name" : "fake"
26+
}
27+
],
28+
"transform" : [],
29+
"sink" : [
30+
{
31+
"plugin_name" : "Clickhouse",
32+
"host" : "localhost:8123",
33+
"username" : "c2VhdHVubmVs",
34+
"password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
35+
"database" : "default",
36+
"table" : "fake_all",
37+
"support_upsert" : true,
38+
"primary_key" : "id"
39+
}
40+
]
41+
}

‎seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,6 @@ public static void buildRequestParams(Map<String, String> requestParams, String
6767

6868
public static Config buildConfig(JsonNode jsonNode, boolean isEncrypt) {
6969
Map<String, Object> objectMap = JsonUtils.toMap(jsonNode);
70-
return ConfigBuilder.of(objectMap, isEncrypt);
70+
return ConfigBuilder.of(objectMap, isEncrypt, true);
7171
}
7272
}

0 commit comments

Comments
 (0)