Add metrics to Athenz{Service,Client} (#6517)

Motivation:

`AthenzService` and `AthenzClient` do not expose metrics that are
helpful to monitor.

Modifications:

- `ZtsBaseClient`)
- The metrics for the underline token fetch client could be enabled by
using
`ZtsBaseClientBuilder.enableMetrics(MeterRegistory,MeterIdPrefixFunction)`
- `AthenzClient` decorator)
  - Added `AthenzClientBuilder` to fluently configure various options.
- `meterIdPrefix` can be configured through the builder; the default is
`armeria.client.athenz`.
    - The results of token-fetch operations are tracked with timers.
- `AthenzService` decorator)
  - The results of access check is recorded in two timers.
- `meterIdPrefix` can be configured through the builder; the default is
`armeria.server.athenz`.

Result:

- You can monitor Athenz’s authentication results and execution time
using the metrics below.
- `armeria.server.athenz{result=(allowed|denied),
resource=<athenz-resource>, action=<athenz-action>}`
- `armeria.client.athenz{result=(success|failure),
domain=<athenz-domain>, roles=<athenz-roles> type=<token-type>}`
- Closes #6423

Author: ikhoon

athenz/src/main/java/com/linecorp/armeria/client/athenz/AthenzClient.java

@@ -16,15 +16,12 @@
 
 package com.linecorp.armeria.client.athenz;
 
-import static java.util.Objects.requireNonNull;
-
 import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
-import com.google.common.collect.ImmutableList;
-
 import com.linecorp.armeria.client.ClientRequestContext;
 import com.linecorp.armeria.client.HttpClient;
 import com.linecorp.armeria.client.SimpleDecoratingHttpClient;
@@ -33,8 +30,13 @@
 import com.linecorp.armeria.common.RequestHeadersBuilder;
 import com.linecorp.armeria.common.annotation.UnstableApi;
 import com.linecorp.armeria.common.athenz.TokenType;
+import com.linecorp.armeria.common.metric.MeterIdPrefix;
+import com.linecorp.armeria.common.metric.MoreMeters;
 import com.linecorp.armeria.common.util.Exceptions;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+
 /**
  * An {@link HttpClient} that adds an Athenz token to the request headers.
  * {@link TokenType#ACCESS_TOKEN} and {@link TokenType#YAHOO_ROLE_TOKEN} are supported.
@@ -53,18 +55,34 @@
  *     .keyPair("/var/lib/athenz/service.key.pem", "/var/lib/athenz/service.cert.pem")
  *     .build();
  *
+ * // Using builder
+ * WebClient
+ *   .builder()
+ *   .decorator(AthenzClient.builder(ztsBaseClient)
+ *                          .domainName("my-domain")
+ *                          .tokenType(TokenType.ROLE_TOKEN)
+ *                          .newDecorator())
+ *   ...
+ *   .build();
+ *
+ * // Or using static factory method
  * WebClient
  *   .builder()
  *   .decorator(AthenzClient.newDecorator(ztsBaseClient, "my-domain",
- *                                        TokenType.ROLE_TOKEN)
+ *                                        TokenType.ROLE_TOKEN))
  *   ...
  *   .build();
  * }</pre>
  */
 @UnstableApi
 public final class AthenzClient extends SimpleDecoratingHttpClient {
 
-    private static final Duration DEFAULT_REFRESH_BEFORE = Duration.ofMinutes(10);
+    /**
+     * Returns a new {@link AthenzClientBuilder} with the specified {@link ZtsBaseClient}.
+     */
+    public static AthenzClientBuilder builder(ZtsBaseClient ztsBaseClient) {
+        return new AthenzClientBuilder(ztsBaseClient);
+    }
 
     /**
      * Returns a new {@link HttpClient} decorator that obtains an Athenz token for the specified domain and
@@ -74,9 +92,12 @@ public final class AthenzClient extends SimpleDecoratingHttpClient {
      * @param domainName the Athenz domain name
      * @param tokenType the type of Athenz token to obtain
      */
-    public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsBaseClient,
-                                                                  String domainName, TokenType tokenType) {
-        return newDecorator(ztsBaseClient, domainName, ImmutableList.of(), tokenType);
+    public static Function<? super HttpClient, AthenzClient> newDecorator(
+            ZtsBaseClient ztsBaseClient, String domainName, TokenType tokenType) {
+        return builder(ztsBaseClient)
+                .domainName(domainName)
+                .tokenType(tokenType)
+                .newDecorator();
     }
 
     /**
@@ -88,10 +109,13 @@ public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsB
      * @param roleName the Athenz role name
      * @param tokenType the type of Athenz token to obtain
      */
-    public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsBaseClient,
-                                                                  String domainName, String roleName,
-                                                                  TokenType tokenType) {
-        return newDecorator(ztsBaseClient, domainName, ImmutableList.of(roleName), tokenType);
+    public static Function<? super HttpClient, AthenzClient> newDecorator(
+            ZtsBaseClient ztsBaseClient, String domainName, String roleName, TokenType tokenType) {
+        return builder(ztsBaseClient)
+                .domainName(domainName)
+                .roleNames(roleName)
+                .tokenType(tokenType)
+                .newDecorator();
     }
 
     /**
@@ -103,10 +127,13 @@ public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsB
      * @param roleNames the list of Athenz role names
      * @param tokenType the type of Athenz token to obtain
      */
-    public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsBaseClient,
-                                                                  String domainName, List<String> roleNames,
-                                                                  TokenType tokenType) {
-        return newDecorator(ztsBaseClient, domainName, roleNames, tokenType, DEFAULT_REFRESH_BEFORE);
+    public static Function<? super HttpClient, AthenzClient> newDecorator(
+            ZtsBaseClient ztsBaseClient, String domainName, List<String> roleNames, TokenType tokenType) {
+        return builder(ztsBaseClient)
+                .domainName(domainName)
+                .roleNames(roleNames)
+                .tokenType(tokenType)
+                .newDecorator();
     }
 
     /**
@@ -119,26 +146,40 @@ public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsB
      * @param tokenType the type of Athenz token to obtain
      * @param refreshBefore the duration before the token expires to refresh it
      */
-    public static Function<HttpClient, AthenzClient> newDecorator(ZtsBaseClient ztsBaseClient,
-                                                                  String domainName, List<String> roleNames,
-                                                                  TokenType tokenType, Duration refreshBefore) {
-        requireNonNull(ztsBaseClient, "ztsBaseClient");
-        requireNonNull(domainName, "domainName");
-        requireNonNull(roleNames, "roleNames");
-        final ImmutableList<String> roleNames0 = ImmutableList.copyOf(roleNames);
-        requireNonNull(tokenType, "tokenType");
-        requireNonNull(refreshBefore, "refreshBefore");
-        return delegate -> new AthenzClient(delegate, ztsBaseClient, domainName, roleNames0,
-                                            tokenType, refreshBefore);
+    public static Function<? super HttpClient, AthenzClient> newDecorator(
+            ZtsBaseClient ztsBaseClient, String domainName, List<String> roleNames,
+            TokenType tokenType, Duration refreshBefore) {
+        return builder(ztsBaseClient)
+                .domainName(domainName)
+                .roleNames(roleNames)
+                .tokenType(tokenType)
+                .refreshBefore(refreshBefore)
+                .newDecorator();
     }
 
     private final TokenType tokenType;
     private final TokenClient tokenClient;
+    private final Timer successTimer;
+    private final Timer failureTimer;
 
-    private AthenzClient(HttpClient delegate, ZtsBaseClient ztsBaseClient, String domainName,
-                         List<String> roleNames, TokenType tokenType, Duration refreshBefore) {
+    AthenzClient(HttpClient delegate, ZtsBaseClient ztsBaseClient, String domainName,
+                 List<String> roleNames, TokenType tokenType, Duration refreshBefore,
+                 MeterIdPrefix meterIdPrefix) {
         super(delegate);
         this.tokenType = tokenType;
+        final MeterRegistry meterRegistry = ztsBaseClient.clientFactory().meterRegistry();
+        final String prefix = meterIdPrefix.name("token.fetch");
+        successTimer = MoreMeters.newTimer(meterRegistry, prefix,
+                                           meterIdPrefix.tags("result", "success",
+                                                              "domain", domainName,
+                                                              "roles", String.join(",", roleNames),
+                                                              "type", tokenType.name()));
+        failureTimer = MoreMeters.newTimer(meterRegistry, prefix,
+                                           meterIdPrefix.tags("result", "failure",
+                                                              "domain", domainName,
+                                                              "roles", String.join(",", roleNames),
+                                                              "type", tokenType.name()));
+
         if (tokenType.isRoleToken()) {
             tokenClient = new RoleTokenClient(ztsBaseClient, domainName, roleNames, refreshBefore);
         } else {
@@ -148,7 +189,16 @@ private AthenzClient(HttpClient delegate, ZtsBaseClient ztsBaseClient, String do
 
     @Override
     public HttpResponse execute(ClientRequestContext ctx, HttpRequest req) throws Exception {
-        final CompletableFuture<HttpResponse> future = tokenClient.getToken().thenApply(token -> {
+        final long startNanos = System.nanoTime();
+
+        final CompletableFuture<HttpResponse> future = tokenClient.getToken().handle((token, cause) -> {
+            final long elapsedNanos = System.nanoTime() - startNanos;
+            if (cause != null) {
+                failureTimer.record(elapsedNanos, TimeUnit.NANOSECONDS);
+                return Exceptions.throwUnsafely(cause);
+            }
+
+            successTimer.record(elapsedNanos, TimeUnit.NANOSECONDS);
             final HttpRequest newReq = req.mapHeaders(headers -> {
                 final RequestHeadersBuilder builder = headers.toBuilder();
                 String token0 = token;

athenz/src/main/java/com/linecorp/armeria/client/athenz/AthenzClientBuilder.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2025 LY Corporation
+ *
+ * LY Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.linecorp.armeria.client.athenz;
+
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.Objects.requireNonNull;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.function.Function;
+
+import com.google.common.collect.ImmutableList;
+
+import com.linecorp.armeria.client.HttpClient;
+import com.linecorp.armeria.common.annotation.Nullable;
+import com.linecorp.armeria.common.annotation.UnstableApi;
+import com.linecorp.armeria.common.athenz.TokenType;
+import com.linecorp.armeria.common.metric.MeterIdPrefix;
+
+/**
+ * A builder for creating an {@link AthenzClient} decorator.
+ *
+ * <p>Example:
+ * <pre>{@code
+ * ZtsBaseClient ztsBaseClient = ...;
+ *
+ * AthenzClient.builder(ztsBaseClient)
+ *             .domainName("my-domain")
+ *             .roleNames("my-role")
+ *             .tokenType(TokenType.ACCESS_TOKEN)
+ *             .refreshBefore(Duration.ofMinutes(5))
+ *             .newDecorator();
+ * }</pre>
+ */
+@UnstableApi
+public final class AthenzClientBuilder {
+
+    private static final Duration DEFAULT_REFRESH_BEFORE = Duration.ofMinutes(10);
+    private static final MeterIdPrefix DEFAULT_METER_ID_PREFIX =
+            new MeterIdPrefix("armeria.client.athenz");
+
+    private final ZtsBaseClient ztsBaseClient;
+    @Nullable
+    private String domainName;
+    private final ImmutableList.Builder<String> roleNamesBuilder = ImmutableList.builder();
+    private TokenType tokenType = TokenType.ACCESS_TOKEN;
+    private Duration refreshBefore = DEFAULT_REFRESH_BEFORE;
+    private MeterIdPrefix meterIdPrefix = DEFAULT_METER_ID_PREFIX;
+
+    AthenzClientBuilder(ZtsBaseClient ztsBaseClient) {
+        this.ztsBaseClient = requireNonNull(ztsBaseClient, "ztsBaseClient");
+    }
+
+    /**
+     * Sets the Athenz domain name.
+     * The domain name must be set before calling {@link #newDecorator()}.
+     */
+    public AthenzClientBuilder domainName(String domainName) {
+        this.domainName = requireNonNull(domainName, "domainName");
+        return this;
+    }
+
+    /**
+     * Adds Athenz role names.
+     */
+    public AthenzClientBuilder roleNames(String... roleNames) {
+        requireNonNull(roleNames, "roleNames");
+        roleNamesBuilder.add(roleNames);
+        return this;
+    }
+
+    /**
+     * Adds Athenz role names.
+     */
+    public AthenzClientBuilder roleNames(Iterable<String> roleNames) {
+        requireNonNull(roleNames, "roleNames");
+        roleNamesBuilder.addAll(roleNames);
+        return this;
+    }
+
+    /**
+     * Sets the type of Athenz token to obtain.
+     * If not set, the default is {@link TokenType#ACCESS_TOKEN}.
+     */
+    public AthenzClientBuilder tokenType(TokenType tokenType) {
+        this.tokenType = requireNonNull(tokenType, "tokenType");
+        return this;
+    }
+
+    /**
+     * Sets the duration before the token expires to refresh it.
+     * If not set, the default is 10 minutes.
+     */
+    public AthenzClientBuilder refreshBefore(Duration refreshBefore) {
+        requireNonNull(refreshBefore, "refreshBefore");
+        checkState(!refreshBefore.isNegative(), "refreshBefore: %s (expected: >= 0)", refreshBefore);
+        this.refreshBefore = refreshBefore;
+        return this;
+    }
+
+    /**
+     * Sets the prefix to use when naming the metrics for this client.
+     * If not set, the default is {@code "armeria.client.athenz"}.
+     */
+    public AthenzClientBuilder meterIdPrefix(MeterIdPrefix meterIdPrefix) {
+        this.meterIdPrefix = requireNonNull(meterIdPrefix, "meterIdPrefix");
+        return this;
+    }
+
+    /**
+     * Returns a new {@link HttpClient} decorator configured with the settings in this builder.
+     */
+    public Function<? super HttpClient, AthenzClient> newDecorator() {
+        final String domainName = this.domainName;
+        checkState(domainName != null, "domainName is not set");
+
+        final List<String> roleNames = roleNamesBuilder.build();
+        return delegate -> new AthenzClient(delegate, ztsBaseClient, domainName, roleNames,
+                                            tokenType, refreshBefore, meterIdPrefix);
+    }
+}
+

athenz/src/main/java/com/linecorp/armeria/client/athenz/ZtsBaseClientBuilder.java

@@ -35,13 +35,17 @@
 import com.linecorp.armeria.client.ClientFactoryBuilder;
 import com.linecorp.armeria.client.WebClient;
 import com.linecorp.armeria.client.WebClientBuilder;
+import com.linecorp.armeria.client.metric.MetricCollectingClient;
 import com.linecorp.armeria.client.proxy.ConnectProxyConfig;
 import com.linecorp.armeria.client.proxy.ProxyConfig;
 import com.linecorp.armeria.common.TlsKeyPair;
 import com.linecorp.armeria.common.annotation.Nullable;
 import com.linecorp.armeria.common.annotation.UnstableApi;
+import com.linecorp.armeria.common.metric.MeterIdPrefixFunction;
 import com.linecorp.armeria.internal.common.util.CertificateUtil;
 
+import io.micrometer.core.instrument.MeterRegistry;
+
 /**
  * A builder for creating a {@link ZtsBaseClient} instance.
  */
@@ -187,6 +191,19 @@ public ZtsBaseClientBuilder configureWebClient(Consumer<? super WebClientBuilder
         return this;
     }
 
+    /**
+     * Enable metrics collection for the ZTS client using the specified {@link MeterRegistry} and
+     * {@link MeterIdPrefixFunction}.
+     */
+    public ZtsBaseClientBuilder enableMetrics(MeterRegistry meterRegistry,
+                                              MeterIdPrefixFunction meterIdPrefixFunction) {
+        requireNonNull(meterRegistry, "meterRegistry");
+        requireNonNull(meterIdPrefixFunction, "meterIdPrefixFunction");
+        configureClientFactory(fb -> fb.meterRegistry(meterRegistry));
+        configureWebClient(cb -> cb.decorator(MetricCollectingClient.newDecorator(meterIdPrefixFunction)));
+        return this;
+    }
+
     /**
      * Builds a new {@link ZtsBaseClient} instance with the configured settings.
      */