Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
031cdd3
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
a9e5f41
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
444af14
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
956e9ee
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
26d454a
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
9283eff
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
7cccc97
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
a427d8d
[Feature][prometheus] Support prometheus read
CosmosNi Jul 24, 2024
ffe6b1c
[Feature][prometheus] Support prometheus read
CosmosNi Jul 25, 2024
f21396c
[Feature][prometheus] Support prometheus read
CosmosNi Jul 25, 2024
e391aa7
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Jul 25, 2024
b62f3f5
[Feature][prometheus] Support prometheus read
CosmosNi Jul 25, 2024
d0c2298
[Feature][prometheus] Support prometheus read
CosmosNi Jul 25, 2024
ce29753
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Aug 9, 2024
705e930
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Aug 13, 2024
d8b5107
[Feature][prometheus] Support prometheus write
CosmosNi Aug 13, 2024
b61857f
[Feature][prometheus] Support prometheus write
CosmosNi Aug 13, 2024
28f9553
[Feature][prometheus] Support prometheus write
CosmosNi Aug 13, 2024
2edc6f2
[Feature][prometheus] Support prometheus write
CosmosNi Aug 14, 2024
01093eb
[Feature][prometheus] Support prometheus write
CosmosNi Aug 14, 2024
0825a0a
[Feature][prometheus] Support prometheus write
CosmosNi Aug 14, 2024
8993a7f
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Aug 14, 2024
04371ee
[Feature][prometheus] Support prometheus write
CosmosNi Aug 14, 2024
5a96dc5
[Feature][prometheus] Support prometheus write
CosmosNi Aug 14, 2024
a4efcd1
[Feature][prometheus] Support prometheus write
CosmosNi Aug 15, 2024
27e5524
[Feature][prometheus] Support prometheus write
CosmosNi Aug 15, 2024
4bbbc17
[Feature][prometheus] Support prometheus write
CosmosNi Aug 15, 2024
a7e647c
[Feature][prometheus] Support prometheus write
CosmosNi Aug 15, 2024
4a96e2b
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Aug 15, 2024
25530a9
[Feature][prometheus] Support prometheus write
CosmosNi Aug 16, 2024
f847b68
[Feature][prometheus] Support prometheus write
CosmosNi Aug 16, 2024
95e7157
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Aug 22, 2024
430da4b
[Feature][prometheus] Support prometheus write
CosmosNi Aug 22, 2024
cde88c9
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Aug 22, 2024
d822c3d
[Feature][prometheus] Support prometheus write
CosmosNi Aug 22, 2024
9903895
[Feature][prometheus] Support prometheus write
CosmosNi Aug 22, 2024
b3f91ea
[Feature][prometheus] Support prometheus write
CosmosNi Aug 22, 2024
4bded40
[Feature][prometheus] Support prometheus write
CosmosNi Aug 22, 2024
88d371a
[Feature][prometheus] Support prometheus write
CosmosNi Aug 29, 2024
be1519b
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Aug 29, 2024
3d054fb
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Aug 30, 2024
894ce5e
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 2, 2024
12d4f9b
[Feature][prometheus] Support prometheus write
CosmosNi Sep 4, 2024
c891145
[Feature][prometheus] Support prometheus write
CosmosNi Sep 4, 2024
9e3a5aa
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 4, 2024
7d1ca3c
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 5, 2024
9b6678c
[Feature][prometheus] Support prometheus write flush interval
CosmosNi Sep 5, 2024
33aebb3
[Feature][prometheus] Modify description
CosmosNi Sep 10, 2024
9e15f0c
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 10, 2024
8acc8a9
[Feature][prometheus] Modify description
CosmosNi Sep 12, 2024
29a398c
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 13, 2024
db8bbe0
[Feature][prometheus] Shade the driver of protobuf to prevent the con…
CosmosNi Sep 13, 2024
788484a
[Feature][prometheus] Shade the driver of protobuf to prevent the con…
CosmosNi Sep 13, 2024
ccda660
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 14, 2024
6e61dd1
[Feature][prometheus] modify e2e
CosmosNi Sep 14, 2024
ab7c7b0
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 18, 2024
2561867
[Feature][prometheus] add vm e2e
CosmosNi Sep 18, 2024
0bd3e63
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 19, 2024
1ca2e67
[Feature][prometheus] modify prometheus source read
CosmosNi Sep 19, 2024
c4bdcff
[Feature][prometheus] modify prometheus source read
CosmosNi Sep 19, 2024
93a916d
[Feature][prometheus] add license
CosmosNi Sep 19, 2024
dbe8bd1
[Feature][prometheus] delete unused class
CosmosNi Sep 19, 2024
a1aa379
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 19, 2024
54736c8
[Feature][prometheus] add time param
CosmosNi Sep 19, 2024
84ab23c
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 20, 2024
489b599
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 20, 2024
908ee50
[Feature][prometheus] add Boundedness.BOUNDED
CosmosNi Sep 20, 2024
fb34df1
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 23, 2024
483418d
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 23, 2024
b9bc999
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 24, 2024
921daed
[Feature][prometheus] support prometheus streaming
CosmosNi Sep 26, 2024
06f3b61
Merge remote-tracking branch 'refs/remotes/upstream/dev' into feature…
CosmosNi Sep 26, 2024
ffc451b
Merge remote-tracking branch 'upstream/dev' into feature-prometheus-s…
CosmosNi Oct 8, 2024
3b9c6e8
[Fix][Connector-V2] Fix prometheus source can not read streaming
CosmosNi Oct 8, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/labeler/label-scope-conf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,11 @@ http:
- changed-files:
- any-glob-to-any-file: seatunnel-connectors-v2/connector-http/**
- all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(http)/**'
prometheus:
- all:
- changed-files:
- any-glob-to-any-file: seatunnel-connectors-v2/connector-prometheus/**
- all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(prometheus)/**'
hudi:
- all:
- changed-files:
Expand Down
3 changes: 2 additions & 1 deletion config/plugin_config
Original file line number Diff line number Diff line change
Expand Up @@ -86,4 +86,5 @@ connector-rocketmq
connector-tdengine
connector-web3j
connector-milvus
--end--
connector-prometheus
--end--
152 changes: 152 additions & 0 deletions docs/en/connector-v2/source/prometheus.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
# Prometheus

> Prometheus source connector

## Description

Used to read data from Prometheus.

## Key features

- [x] [batch](../../concept/connector-v2-features.md)
- [ ] [stream](../../concept/connector-v2-features.md)
- [ ] [exactly-once](../../concept/connector-v2-features.md)
- [ ] [column projection](../../concept/connector-v2-features.md)
- [ ] [parallelism](../../concept/connector-v2-features.md)
- [ ] [support user-defined split](../../concept/connector-v2-features.md)

## Options

| name | type | required | default value |
|-----------------------------|---------|----------|-----------------|
| url | String | Yes | - |
| query | String | Yes | - |
| query_type | String | Yes | Instant |
| content_field | String | Yes | $.data.result.* |
| schema.fields | Config | Yes | - |
| format | String | No | json |
| params | Map | Yes | - |
| poll_interval_millis | int | No | - |
| retry | int | No | - |
| retry_backoff_multiplier_ms | int | No | 100 |
| retry_backoff_max_ms | int | No | 10000 |
| enable_multi_lines | boolean | No | false |
| common-options | config | No | - |

### url [String]

http request url

### query [String]

Prometheus expression query string

### query_type [String]

Instant/Range

1. Instant : The following endpoint evaluates an instant query at a single point in time
2. Range : The following endpoint evaluates an expression query over a range of time

https://prometheus.io/docs/prometheus/latest/querying/api/

### params [Map]

http params

### poll_interval_millis [int]

request http api interval(millis) in stream mode

### retry [int]

The max retry times if request http return to `IOException`

### retry_backoff_multiplier_ms [int]

The retry-backoff times(millis) multiplier if request http failed

### retry_backoff_max_ms [int]

The maximum retry-backoff times(millis) if request http failed

### format [String]

the format of upstream data, default `json`.

### schema [Config]

Fill in a fixed value

```hocon
schema = {
fields {
metric = "map<string, string>"
value = "array<int>"
}
}

```

#### fields [Config]

the schema fields of upstream data

### common options

Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details

## Example

### Instant:

```hocon
source {
Prometheus {
result_table_name = "http"
url = "http://mockserver:1080"
query = "up"
query_type = "Instant"
content_field = "$.data.result.*"
format = "json"
schema = {
fields {
metric = "map<string, string>"
value = "array<int>"
}
}
}
}
```

### Range

```hocon
source {
Prometheus {
result_table_name = "http"
url = "http://mockserver:1080"
query = "up"
query_type = "Range"
content_field = "$.data.result.*"
format = "json"
start = "2024-07-22T20:10:30.781Z"
end = "2024-07-22T20:11:00.781Z"
step = "15s"
schema = {
fields {
metric = "map<string, string>"
value = "array<int>"
}
}
}
}
```

## Changelog

### next version

- Add Prometheus Source Connector
- Reduce configuration items

1 change: 1 addition & 0 deletions plugin-mapping.properties
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,4 @@ seatunnel.source.ObsFile = connector-file-obs
seatunnel.sink.ObsFile = connector-file-obs
seatunnel.source.Milvus = connector-milvus
seatunnel.sink.Milvus = connector-milvus
seatunnel.source.Prometheus = connector-prometheus
58 changes: 58 additions & 0 deletions seatunnel-connectors-v2/connector-prometheus/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-connectors-v2</artifactId>
<version>${revision}</version>
</parent>

<artifactId>connector-prometheus</artifactId>
<name>SeaTunnel : Connectors V2 : Prometheus</name>

<properties>
<prometheus-client.version>0.16.0</prometheus-client.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-http-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>${prometheus-client.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>${prometheus-client.version}</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.seatunnel.connectors.seatunnel.prometheus.config;

import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options;
import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig;

/**
* SourceConfig is the configuration for the PrometheusSource.
*
* <p>please see the following link for more details:
* https://prometheus.io/docs/prometheus/latest/querying/api/
*/
public class PrometheusSourceConfig extends HttpConfig {

public static final String INSTANT_QUERY_URL = "/api/v1/query";

public static final String RANGE_QUERY = "Range";

public static final String RANGE_QUERY_URL = "/api/v1/query_range";

public static final Option<String> QUERY =
Options.key("query")
.stringType()
.noDefaultValue()
.withDescription("Prometheus expression query string");

public static final Option<String> QUERY_TYPE =
Options.key("query_type")
.stringType()
.defaultValue("Instant")
.withDescription("Prometheus expression query string");

public static final Option<Long> TIMEOUT =
Options.key("timeout")
.longType()
.noDefaultValue()
.withDescription("Evaluation timeout");

public static class RangeConfig {

public static final Option<String> START =
Options.key("start")
.stringType()
.noDefaultValue()
.withDescription("Start timestamp, inclusive.");

public static final Option<String> END =
Options.key("end")
.stringType()
.noDefaultValue()
.withDescription("End timestamp, inclusive.");

public static final Option<String> STEP =
Options.key("step")
.stringType()
.noDefaultValue()
.withDescription(
" Query resolution step width in duration format or float number of seconds.");
}

public static class InstantQueryConfig {
public static final Option<Long> TIME =
Options.key("time")
.longType()
.noDefaultValue()
.withDescription("Evaluation timestamp,unix_timestamp");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.seatunnel.connectors.seatunnel.prometheus.config;

import org.apache.seatunnel.shade.com.typesafe.config.Config;

import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod;

import java.util.HashMap;

import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.INSTANT_QUERY_URL;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.InstantQueryConfig.TIME;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.QUERY;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.QUERY_TYPE;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.RANGE_QUERY;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.RANGE_QUERY_URL;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.RangeConfig.END;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.RangeConfig.START;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.RangeConfig.STEP;
import static org.apache.seatunnel.connectors.seatunnel.prometheus.config.PrometheusSourceConfig.TIMEOUT;

public class PrometheusSourceParameter extends HttpParameter {
public void buildWithConfig(Config pluginConfig) {
super.buildWithConfig(pluginConfig);

String query = pluginConfig.getString(QUERY.key());

String queryType =
pluginConfig.hasPath(QUERY_TYPE.key())
? pluginConfig.getString(QUERY_TYPE.key())
: QUERY_TYPE.defaultValue();

this.params = this.getParams() == null ? new HashMap<>() : this.getParams();

params.put(PrometheusSourceConfig.QUERY.key(), query);

this.setMethod(HttpRequestMethod.GET);

if (pluginConfig.hasPath(TIMEOUT.key())) {
params.put(TIMEOUT.key(), pluginConfig.getString(TIMEOUT.key()));
}

if (RANGE_QUERY.equals(queryType)) {
this.setUrl(this.getUrl() + RANGE_QUERY_URL);
params.put(START.key(), pluginConfig.getString(START.key()));
params.put(END.key(), pluginConfig.getString(END.key()));
params.put(STEP.key(), pluginConfig.getString(STEP.key()));

} else {
this.setUrl(this.getUrl() + INSTANT_QUERY_URL);
if (pluginConfig.hasPath(TIME.key())) {
String time = pluginConfig.getString(TIME.key());
params.put(TIME.key(), time);
}
}
this.setParams(params);
}
}
Loading