Skip to content

Commit 38713f6

Browse files
hawk9821hawk9821
committed
[improve] Handle user privacy when submitting a task print configuration logs
[improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs [improve] Handle user privacy when submitting a task print configuration logs
1 parent 825caa8 commit 38713f6

File tree

6 files changed

+187
-7
lines changed

6 files changed

+187
-7
lines changed

‎config/seatunnel.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ seatunnel:
3333
plugin-config:
3434
namespace: /tmp/seatunnel/checkpoint_snapshot
3535
storage.type: hdfs
36-
fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
36+
fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission

‎seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java‎

Lines changed: 98 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,23 @@
2525
import org.apache.seatunnel.shade.com.typesafe.config.impl.Parseable;
2626

2727
import org.apache.seatunnel.api.configuration.ConfigAdapter;
28+
import org.apache.seatunnel.common.utils.JsonUtils;
2829
import org.apache.seatunnel.common.utils.ParserException;
2930

3031
import lombok.NonNull;
3132
import lombok.extern.slf4j.Slf4j;
3233

3334
import java.nio.file.Path;
3435
import java.nio.file.Paths;
36+
import java.util.Arrays;
37+
import java.util.HashMap;
3538
import java.util.List;
3639
import java.util.Map;
3740
import java.util.Objects;
3841
import java.util.Optional;
42+
import java.util.stream.Collectors;
43+
44+
import static org.apache.seatunnel.core.starter.utils.ConfigShadeUtils.DEFAULT_SENSITIVE_KEYWORDS;
3945

4046
/** Used to build the {@link Config} from config file. */
4147
@Slf4j
@@ -76,14 +82,19 @@ public static Config of(@NonNull Path filePath, List<String> variables) {
7682
adapterSupplier
7783
.map(adapter -> of(adapter, filePath, variables))
7884
.orElseGet(() -> ofInner(filePath, variables));
85+
boolean isJson = filePath.getFileName().toString().endsWith(".json");
86+
log.info(
87+
"Parsed config file: \n{}",
88+
mapToString(configDesensitization(config.root().unwrapped()), isJson));
7989
return config;
8090
}
8191

8292
public static Config of(@NonNull Map<String, Object> objectMap) {
83-
return of(objectMap, false);
93+
return of(objectMap, false, false);
8494
}
8595

86-
public static Config of(@NonNull Map<String, Object> objectMap, boolean isEncrypt) {
96+
public static Config of(
97+
@NonNull Map<String, Object> objectMap, boolean isEncrypt, boolean isJson) {
8798
log.info("Loading config file from objectMap");
8899
Config config =
89100
ConfigFactory.parseMap(objectMap)
@@ -94,9 +105,50 @@ public static Config of(@NonNull Map<String, Object> objectMap, boolean isEncryp
94105
if (!isEncrypt) {
95106
config = ConfigShadeUtils.decryptConfig(config);
96107
}
108+
log.info(
109+
"Parsed config file: \n{}",
110+
mapToString(configDesensitization(config.root().unwrapped()), isJson));
97111
return config;
98112
}
99113

114+
public static Map<String, Object> configDesensitization(Map<String, Object> configMap) {
115+
return configMap.entrySet().stream()
116+
.collect(
117+
Collectors.toMap(
118+
Map.Entry::getKey,
119+
e -> {
120+
String key = e.getKey();
121+
if (Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)
122+
.contains(key.toLowerCase())) {
123+
return "******";
124+
} else {
125+
Object value = e.getValue();
126+
if (value instanceof Map) {
127+
if ("schema".equals(key)) {
128+
return value;
129+
}
130+
return configDesensitization(
131+
(Map<String, Object>) value);
132+
} else if (value instanceof List) {
133+
List<Object> list = (List<Object>) value;
134+
return list.stream()
135+
.map(
136+
v -> {
137+
if (v instanceof Map) {
138+
return configDesensitization(
139+
(Map<String, Object>)
140+
v);
141+
} else {
142+
return v;
143+
}
144+
})
145+
.collect(Collectors.toList());
146+
}
147+
return value;
148+
}
149+
}));
150+
}
151+
100152
public static Config of(
101153
@NonNull ConfigAdapter configAdapter, @NonNull Path filePath, List<String> variables) {
102154
log.info("With config adapter spi {}", configAdapter.getClass().getName());
@@ -133,4 +185,48 @@ private static Config backfillUserVariables(Config config, List<String> variable
133185
}
134186
return config;
135187
}
188+
189+
public static String mapToString(Map<String, Object> configMap, boolean isJson) {
190+
ConfigRenderOptions configRenderOptions =
191+
ConfigRenderOptions.concise().setFormatted(true).setJson(isJson);
192+
if (!isJson) {
193+
convertHoconMap(configMap);
194+
}
195+
Config config =
196+
ConfigFactory.parseString(JsonUtils.toJsonString(configMap))
197+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
198+
.resolveWith(
199+
ConfigFactory.systemProperties(),
200+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
201+
return config.root().render(configRenderOptions);
202+
}
203+
204+
private static void convertHoconMap(Map<String, Object> configMap) {
205+
convertField(configMap, "source");
206+
convertField(configMap, "sink");
207+
}
208+
209+
private static void convertField(Map<String, Object> configMap, String fieldName) {
210+
if (configMap.containsKey(fieldName)) {
211+
Object fieldValue = configMap.get(fieldName);
212+
if (fieldValue instanceof List) {
213+
@SuppressWarnings("unchecked")
214+
List<Map<String, Object>> list = (List<Map<String, Object>>) fieldValue;
215+
Map<String, Object> newMap =
216+
list.stream()
217+
.collect(
218+
HashMap::new,
219+
(m, entry) -> {
220+
String pluginName =
221+
entry.getOrDefault("plugin_name", "")
222+
.toString();
223+
Map<String, Object> pluginConfig = new HashMap<>(entry);
224+
pluginConfig.remove("plugin_name");
225+
m.put(pluginName, pluginConfig);
226+
},
227+
HashMap::putAll);
228+
configMap.put(fieldName, newMap);
229+
}
230+
}
231+
}
136232
}

‎seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ public final class ConfigShadeUtils {
4747

4848
private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier";
4949

50-
private static final String[] DEFAULT_SENSITIVE_OPTIONS =
51-
new String[] {"password", "username", "auth"};
50+
public static final String[] DEFAULT_SENSITIVE_KEYWORDS =
51+
new String[] {"password", "username", "auth", "token"};
5252

5353
private static final Map<String, ConfigShade> CONFIG_SHADES = new HashMap<>();
5454

@@ -126,7 +126,7 @@ public static Config encryptConfig(String identifier, Config config) {
126126
@SuppressWarnings("unchecked")
127127
private static Config processConfig(String identifier, Config config, boolean isDecrypted) {
128128
ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE);
129-
List<String> sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_OPTIONS));
129+
List<String> sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS));
130130
sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions()));
131131
BiFunction<String, Object, String> processFunction =
132132
(key, value) -> {

‎seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,17 @@
1919

2020
import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
2121
import org.apache.seatunnel.shade.com.typesafe.config.Config;
22+
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
2223
import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject;
24+
import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
2325

2426
import org.apache.seatunnel.api.configuration.ConfigShade;
2527
import org.apache.seatunnel.common.utils.JsonUtils;
2628

2729
import org.junit.jupiter.api.Assertions;
2830
import org.junit.jupiter.api.Test;
2931

32+
import com.beust.jcommander.internal.Lists;
3033
import lombok.extern.slf4j.Slf4j;
3134

3235
import java.net.URISyntaxException;
@@ -68,6 +71,46 @@ public void testParseConfig() throws URISyntaxException {
6871
config.getConfigList("source").get(0).getString("password"), PASSWORD);
6972
}
7073

74+
@Test
75+
public void testUsePrivacyHandlerHocon() throws URISyntaxException {
76+
URL resource = ConfigShadeTest.class.getResource("/config.shade.conf");
77+
Assertions.assertNotNull(resource);
78+
Config config = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList());
79+
config =
80+
ConfigFactory.parseMap(
81+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
82+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
83+
.resolveWith(
84+
ConfigFactory.systemProperties(),
85+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
86+
Assertions.assertEquals(
87+
config.getConfigList("source").get(0).getString("username"), "******");
88+
Assertions.assertEquals(
89+
config.getConfigList("source").get(0).getString("password"), "******");
90+
String conf = ConfigBuilder.mapToString(config.root().unwrapped(), false);
91+
Assertions.assertTrue(conf.contains("username=\"******\""));
92+
}
93+
94+
@Test
95+
public void testUsePrivacyHandlerJson() throws URISyntaxException {
96+
URL resource = ConfigShadeTest.class.getResource("/config.shade.json");
97+
Assertions.assertNotNull(resource);
98+
Config config = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList());
99+
config =
100+
ConfigFactory.parseMap(
101+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
102+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
103+
.resolveWith(
104+
ConfigFactory.systemProperties(),
105+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
106+
Assertions.assertEquals(
107+
config.getConfigList("source").get(0).getString("username"), "******");
108+
Assertions.assertEquals(
109+
config.getConfigList("source").get(0).getString("password"), "******");
110+
String json = ConfigBuilder.mapToString(config.root().unwrapped(), true);
111+
Assertions.assertTrue(json.contains("\"password\" : \"******\""));
112+
}
113+
71114
@Test
72115
public void testVariableReplacement() throws URISyntaxException {
73116
String jobName = "seatunnel variable test job";

‎seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"env" : {
3+
"shade.identifier" : "base64",
4+
"parallelism" : 1
5+
},
6+
"source" : [
7+
{
8+
"plugin_name" : "MySQL-CDC",
9+
"base-url" : "jdbc:mysql://localhost:56725",
10+
"username" : "c2VhdHVubmVs",
11+
"password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
12+
"hostname" : "127.0.0.1",
13+
"port" : 56725,
14+
"database-name" : "inventory_vwyw0n",
15+
"parallelism" : 1,
16+
"table-name" : "products",
17+
"server-id" : 5656,
18+
"schema" : {
19+
"fields" : {
20+
"name" : "string",
21+
"age" : "int",
22+
"sex" : "boolean"
23+
}
24+
},
25+
"result_table_name" : "fake"
26+
}
27+
],
28+
"transform" : [],
29+
"sink" : [
30+
{
31+
"plugin_name" : "Clickhouse",
32+
"host" : "localhost:8123",
33+
"username" : "c2VhdHVubmVs",
34+
"password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
35+
"database" : "default",
36+
"table" : "fake_all",
37+
"support_upsert" : true,
38+
"primary_key" : "id"
39+
}
40+
]
41+
}

‎seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,6 @@ public static void buildRequestParams(Map<String, String> requestParams, String
6767

6868
public static Config buildConfig(JsonNode jsonNode, boolean isEncrypt) {
6969
Map<String, Object> objectMap = JsonUtils.toMap(jsonNode);
70-
return ConfigBuilder.of(objectMap, isEncrypt);
70+
return ConfigBuilder.of(objectMap, isEncrypt, true);
7171
}
7272
}

0 commit comments

Comments
 (0)