diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 48a0e5b0ef3..beab2f488a8 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -27,6 +27,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedActionException; @@ -319,7 +320,7 @@ protected GssApiAuthenticator( SUPPORTED_MECHANISMS, options.getAuthorizationId(), protocol, - ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(), + serverName(endPoint), options.getSaslProperties(), null); } catch (LoginException | SaslException e) { @@ -328,6 +329,23 @@ protected GssApiAuthenticator( this.endPoint = endPoint; } + /** + * The host name to build the Kerberos service principal from. + * + *

Prefers the canonical name of the resolved address, which is what Kerberos expects. The + * driver's own endpoints always hand this a resolved address — the channel carries an endpoint + * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link + * EndPoint} implementation may still yield an unresolved one, in which case {@code + * getAddress()} is null. Fall back to the host string rather than throwing a {@link + * NullPointerException}: the hostname is usually the right service name anyway, and a failed + * reverse lookup should not take authentication down. + */ + private static String serverName(EndPoint endPoint) { + InetSocketAddress address = (InetSocketAddress) endPoint.resolve(); + InetAddress inetAddress = address.getAddress(); + return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString(); + } + @NonNull @Override protected ByteBuffer getMechanism() { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index dd60a2487fb..da0a3c622cb 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -135,6 +135,13 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: int */ CONNECTION_MAX_ORPHAN_REQUESTS("advanced.connection.max-orphan-requests"), + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + * + *

Value-type: int + */ + CONNECTION_MAX_CANDIDATE_ADDRESSES("advanced.connection.max-candidate-addresses"), /** * Whether to log non-fatal errors when the driver tries to open a new connection. * @@ -701,8 +708,14 @@ public enum DefaultDriverOption implements DriverOption { CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"), /** - * Whether to forcibly add original contact points held by MetadataManager to the reconnection - * plan, in case there is no live nodes available according to LBP. Experimental. + * Whether to append the original contact points held by MetadataManager to the reconnection plan, + * after the live nodes reported by the load balancing policy. Defaults to {@code true}. + * + *

This is also the driver's DNS re-resolution path. Contact points are appended as-is, still + * unresolved hostnames, and each is expanded to its current DNS IPs at connection time through + * Netty's configured resolver. Metadata nodes, in contrast, hold an already-resolved endpoint + * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve + * the original hostnames and pick up new IPs once the live-node plan is exhausted. * *

Value-type: boolean */ @@ -837,7 +850,14 @@ public enum DefaultDriverOption implements DriverOption { * Whether to resolve the addresses passed to `basic.contact-points`. * *

Value-type: boolean + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. */ + @Deprecated RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"), /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index c1a428b3524..bc5bf51c2ba 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException throw new InvalidObjectException("Proxy required"); } + // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the + // defaults map stays complete by carrying its reference.conf value. + @SuppressWarnings("deprecation") protected static void fillWithDriverDefaults(OptionsMap map) { Duration initQueryTimeout = Duration.ofSeconds(5); Duration requestTimeout = Duration.ofSeconds(2); @@ -276,6 +279,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONNECTION_POOL_INIT_BATCH_SIZE, 0); map.put(TypedDriverOption.CONNECTION_MAX_REQUESTS, 1024); map.put(TypedDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, 256); + map.put(TypedDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, 5); map.put(TypedDriverOption.CONNECTION_WARN_INIT_ERROR, true); map.put(TypedDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED, true); map.put(TypedDriverOption.ADVANCED_SHARD_AWARENESS_PORT_LOW, 10000); @@ -369,7 +373,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true); - map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false); + map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true); map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true); map.put(TypedDriverOption.REPREPARE_ENABLED, true); map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index af93e734ef1..fe7888682bc 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -172,6 +172,13 @@ public String toString() { public static final TypedDriverOption CONNECTION_MAX_ORPHAN_REQUESTS = new TypedDriverOption<>( DefaultDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, GenericType.INTEGER); + /** + * The maximum number of addresses a single connection attempt will try, when the endpoint it + * connects to is a DNS name that resolves to several addresses. + */ + public static final TypedDriverOption CONNECTION_MAX_CANDIDATE_ADDRESSES = + new TypedDriverOption<>( + DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, GenericType.INTEGER); /** Whether to log non-fatal errors when the driver tries to open a new connection. */ public static final TypedDriverOption CONNECTION_WARN_INIT_ERROR = new TypedDriverOption<>(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR, GenericType.BOOLEAN); @@ -600,7 +607,15 @@ public String toString() { public static final TypedDriverOption CONTROL_CONNECTION_AGREEMENT_WARN = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN); - /** Whether to forcibly try original contacts if no live nodes are available */ + /** + * Whether to append the original contact points to the control-connection reconnection plan, + * after the live nodes reported by the load balancing policy (defaults to {@code true}). + * + *

Contact points are appended as-is (unresolved hostnames); each is expanded to all of its + * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The + * append is skipped for topology monitors that re-resolve node addresses themselves (such as the + * cloud/proxy monitors). + */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN); @@ -664,7 +679,16 @@ public String toString() { /** The coalescer reschedule interval. */ public static final TypedDriverOption COALESCER_INTERVAL = new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION); - /** Whether to resolve the addresses passed to `basic.contact-points`. */ + /** + * Whether to resolve the addresses passed to `basic.contact-points`. + * + * @deprecated Setting this option has no effect. Contact points given in the configuration are + * now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily + * at connection time. This never applied to programmatic contact points passed to {@code + * SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved + * address stays bound to that one IP. + */ + @Deprecated public static final TypedDriverOption RESOLVE_CONTACT_POINTS = new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN); /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index 530f2ad38ac..06dea1daef2 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -18,24 +18,59 @@ package com.datastax.oss.driver.api.core.metadata; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetSocketAddress; import java.net.SocketAddress; /** * Encapsulates the information needed to open connections to a node. * *

By default, the driver assumes plain TCP connections, and this is just a wrapper around an - * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom + * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom * implementation that contains additional information; for example, if the nodes are accessed * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address. */ public interface EndPoint { /** - * Resolves this instance to a socket address. + * Resolves this instance to the socket address connections should be opened to. * *

This will be called each time the driver opens a new connection to the node. The returned * address cannot be null. + * + *

Returning a hostname is fine, and is how multi-address support works. The returned + * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved() + * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to every + * address the name maps to, and each one is tried in turn until a connection succeeds. That is + * what {@link com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does for contact + * points backed by a hostname, so a single unreachable IP behind a multi-record name no longer + * fails the connection. + * + *

Implementations must not resolve names themselves, and must not block. The driver + * calls this from its admin event loop, and it performs the expansion through Netty's configured + * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is + * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with + * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a + * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Callers must not assume the returned address is resolved. It is for a node discovered + * from {@code system.peers} (built from that node's physical broadcast RPC address) and for the + * node the control connection is on (bound to the address that connection reached). It is + * not for a node reached through the Cloud SNI proxy, or through a cloud private-endpoint + * client route: there the address is the configured hostname, and {@link + * java.net.InetSocketAddress#getAddress()} returns {@code null}. Read the host with {@link + * java.net.InetSocketAddress#getHostString()}, which yields whichever of the two the address + * carries and never triggers a reverse lookup. + * + * @apiNote Timeout note: when a name expands to several addresses they are tried in + * sequence, so the worst-case time before the node is declared unreachable is N times a full + * attempt — and an attempt is more than a connect. Each address that accepts the TCP + * connection then runs the init handshake, whose steps each arm their own {@code + * advanced.connection.init-query-timeout}; those add up rather than sharing one deadline. An + * address that stalls after accepting the connection can therefore burn {@code + * advanced.connection.connect-timeout} plus several times {@code + * advanced.connection.init-query-timeout} on its own. In practice DNS round-robin entries + * have only a small number of records, so this is rarely a concern, but it is worth bearing + * in mind when configuring timeouts — note also that session initialization has no overall + * deadline of its own. */ @NonNull SocketAddress resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 8375f0ef30b..409ac5a589f 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad *

Contact points can also be provided statically in the configuration. If both are specified, * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042. * - *

Contrary to the configuration, DNS names with multiple A-records will not be handled here. - * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)} - * before calling this method. Similarly, if you need connect addresses to stay unresolved, make - * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the - * configuration for more explanations). + *

The driver automatically expands any contact point backed by an unresolved hostname to all + * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code + * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its + * IPs on initial connect. This applies equally to hostnames provided here programmatically (build + * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String, + * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address + * passed here (the common case when constructing an {@code InetSocketAddress} directly from a + * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code + * advanced.resolve-contact-points} option is deprecated and has no effect. */ @NonNull public SelfT addContactPoints(@NonNull Collection contactPoints) { @@ -741,6 +745,23 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS * *

For more information, please refer to the DataStax Astra documentation. * + *

A proxy given as a hostname is resolved at connection time, to all of its addresses, + * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the + * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code + * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that + * single address. + * + *

Prefer {@link InetSocketAddress#createUnresolved(String, int)} all the same, and especially + * for a proxy given as an IP address. Whether an address carries a name is read from + * {@code getHostString()}, which falls back to the underlying {@link java.net.InetAddress}'s + * cached host name -- and that field is filled in, on the very instance passed here, the first + * time anything calls {@code getHostName()} on it. The SNI SSL engine does exactly that while + * building an engine, unless reverse-lookup SANs are turned off. So an IP that has a {@code PTR} + * record can acquire a name mid-session, after which the driver treats that name as the proxy: + * the endpoints it builds from then on compare unequal to the earlier ones, report metrics under + * a different prefix, and connect to wherever that name resolves. An unresolved address is never + * subject to this, and is what the secure connect bundle produces. + * * @param cloudProxyAddress The address of the Cloud proxy to use. * @see Server Name Indication */ @@ -957,11 +978,15 @@ protected final CompletionStage buildDefaultSessionAsync() { programmaticArguments = programmaticArgumentsBuilder.build(); } - boolean resolveAddresses = - defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false); - + // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved + // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory. + // The value is still read, only to tell someone who set it that it no longer does anything. + // Note this tests the value rather than isDefined(): unlike the deprecated options warned + // about in DefaultDriverContext, this one ships uncommented in reference.conf, so it is + // always defined. + warnIfResolveContactPointsRequested(defaultConfig); Set contactPoints = - ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses); + ContactPoints.merge(programmaticContactPoints, configContactPoints, false); if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) { keyspace = @@ -989,6 +1014,24 @@ private boolean anyProfileHasDatacenterDefined(DriverConfig driverConfig) { return false; } + /** + * Tells anyone who turned {@code advanced.resolve-contact-points} on that it no longer does + * anything, so the behaviour they configured does not disappear in silence. + */ + @SuppressWarnings("deprecation") + private static void warnIfResolveContactPointsRequested(DriverExecutionProfile defaultConfig) { + if (defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false)) { + LOG.warn( + "Option {} is deprecated and no longer has any effect. Contact points given in the" + + " configuration are now always kept as unresolved hostnames and expanded to all of" + + " their addresses at connection time, so a name that resolves to several nodes is" + + " one Node in the driver's metadata rather than one per address. Note that this" + + " never applied to contact points passed to addContactPoints(): those are used" + + " exactly as supplied, and an already-resolved address stays bound to that one IP.", + DefaultDriverOption.RESOLVE_CONTACT_POINTS.getPath()); + } + } + /** * Returns URL based on the configUrl setting. If the configUrl has no protocol provided, the * method will fallback to file:// protocol and return URL that has file protocol specified. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java index 5078428c21a..02496d68ae1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/adminrequest/AdminRequestHandler.java @@ -30,8 +30,9 @@ import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.request.Query; +import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.query.QueryOptions; -import com.datastax.oss.protocol.internal.response.Result; +import com.datastax.oss.protocol.internal.response.Ready; import com.datastax.oss.protocol.internal.response.result.Prepared; import com.datastax.oss.protocol.internal.response.result.Rows; import io.netty.util.concurrent.Future; @@ -67,6 +68,24 @@ public static AdminRequestHandler call( com.datastax.oss.protocol.internal.response.result.Void.class); } + /** + * Registers this connection for the given protocol events, as {@link + * com.datastax.oss.protocol.internal.request.Register REGISTER} used to be sent as the last step + * of protocol initialization. + */ + public static AdminRequestHandler register( + DriverChannel channel, List eventTypes, Duration timeout, String logPrefix) { + return new AdminRequestHandler<>( + channel, + true, + new Register(eventTypes), + Frame.NO_PAYLOAD, + timeout, + logPrefix, + "register for events " + eventTypes, + Ready.class); + } + public static AdminRequestHandler query( DriverChannel channel, String query, @@ -98,7 +117,7 @@ public static AdminRequestHandler query( private final Duration timeout; private final String logPrefix; private final String debugString; - private final Class expectedResponseType; + private final Class expectedResponseType; protected final CompletableFuture result = new CompletableFuture<>(); // This is only ever accessed on the channel's event loop, so it doesn't need to be volatile @@ -112,7 +131,7 @@ protected AdminRequestHandler( Duration timeout, String logPrefix, String debugString, - Class expectedResponseType) { + Class expectedResponseType) { this.channel = channel; this.shouldPreAcquireId = shouldPreAcquireId; this.message = message; @@ -190,8 +209,9 @@ public void onResponse(Frame responseFrame) { @SuppressWarnings("unchecked") ResultT result = (ResultT) ByteBuffer.wrap(prepared.preparedQueryId); setFinalResult(result); - } else if (expectedResponseType - == com.datastax.oss.protocol.internal.response.result.Void.class) { + } else if (expectedResponseType == com.datastax.oss.protocol.internal.response.result.Void.class + || expectedResponseType == Ready.class) { + // Neither carries a payload: a schema change or a REGISTER acknowledgement. setFinalResult(null); } else { setFinalError(new AssertionError("Unhandled response type" + expectedResponseType)); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 35190afa3f4..7e696fe7d7c 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -24,29 +24,39 @@ package com.datastax.oss.driver.internal.core.channel; import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.InvalidKeyspaceException; import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.context.DriverContext; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeShardingInfo; import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; +import com.datastax.oss.driver.internal.core.adminrequest.UnexpectedResponseException; import com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverConfig; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; import com.datastax.oss.driver.internal.core.protocol.FrameDecoder; import com.datastax.oss.driver.internal.core.protocol.FrameEncoder; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.protocol.internal.Message; +import com.datastax.oss.protocol.internal.ProtocolConstants; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -54,16 +64,33 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoop; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.ScheduledFuture; import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.ServerSocket; import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; @@ -127,6 +154,19 @@ public static int effectiveMaxOrphanRequests( private final String logPrefix; protected final InternalDriverContext context; + /** + * Guards the one-time warning in {@link #newBootstrap()}. Per factory rather than per JVM: what + * it reports is a property of this session's {@link NettyOptions}, and the message names the + * session, so a JVM-wide latch would report the first offender and silence every one after it. + */ + private final AtomicBoolean loggedHandlerWarning = new AtomicBoolean(); + + /** + * Randomizes the order in which a name's expanded addresses are tried (see {@link + * #shuffleAndLimit}). Injectable so tests can seed it and observe a deterministic order. + */ + @VisibleForTesting Random random = new Random(); + /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -145,6 +185,7 @@ public ChannelFactory(InternalDriverContext context) { this.context = context; DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); + if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); this.protocolVersion = context.getProtocolVersionRegistry().fromName(versionName); @@ -181,7 +222,7 @@ public CompletionStage connect(Node node, DriverChannelOptions op } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater); + return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater, isIdentified(node)); } public CompletionStage connect( @@ -192,7 +233,31 @@ public CompletionStage connect( } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), node.getShardingInfo(), shardId, options, nodeMetricUpdater); + return connect( + node.getEndPoint(), + node.getShardingInfo(), + shardId, + options, + nodeMetricUpdater, + isIdentified(node)); + } + + /** + * Whether we know which node we are connecting to, as opposed to merely which address to + * try. {@link #tryNextCandidate} needs the distinction because an unidentified contact-point name + * may expand to addresses of different nodes, while every address of an identified node is + * that same node. + * + *

{@link Node#getHostId()} is null exactly for a contact point, and stays null for the life of + * that instance: {@code MetadataManager.registerNode} mints a fresh {@code DefaultNode} from each + * {@code system.local}/{@code system.peers} row rather than back-filling the contact point it was + * reached through, and those ephemeral contact-point nodes are never added to metadata. So this + * is not a state a contact point grows out of once the driver has read host ids -- the driver + * simply stops using that instance for anything except the reconnection fallback, which keeps + * handing it back. + */ + private static boolean isIdentified(Node node) { + return node.getHostId() != null; } @VisibleForTesting @@ -202,11 +267,23 @@ CompletionStage connect( Integer shardId, DriverChannelOptions options, NodeMetricUpdater nodeMetricUpdater) { + // A bare endpoint carries no host id, so this matches the contact-point case (see + // isIdentified()). + return connect(endPoint, shardingInfo, shardId, options, nodeMetricUpdater, false); + } + + @VisibleForTesting + CompletionStage connect( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + boolean nodeIsIdentified) { CompletableFuture resultFuture = new CompletableFuture<>(); ProtocolVersion currentVersion; boolean isNegotiating; - List attemptedVersions = new CopyOnWriteArrayList<>(); if (this.protocolVersion != null) { currentVersion = protocolVersion; isNegotiating = false; @@ -223,7 +300,7 @@ CompletionStage connect( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, + nodeIsIdentified, resultFuture); return resultFuture; } @@ -236,121 +313,1202 @@ private void connect( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, - List attemptedVersions, + boolean nodeIsIdentified, CompletableFuture resultFuture) { - SocketAddress resolvedAddress; + // Built once per connect() rather than once per candidate: it is the only handle on the Netty + // AddressResolverGroup (see resolveCandidates()), and it means the user's + // afterBootstrapInitialized() hook runs once per logical connection instead of once per address + // attempt. Each attempt gets its own clone() with its own handler. + // + // The event loop is likewise picked once per connect() and shared by name resolution and the + // channel itself (the per-attempt clones are bound to it, see connectToAddress()). Advancing + // the group's round-robin chooser exactly once per connect keeps channels evenly distributed: + // taking one loop for resolution and letting Bootstrap.connect() take another would advance + // the chooser twice per connect, parking all channels on half the loops with the default + // power-of-two chooser. It also mirrors what Netty itself does with an unresolved address: + // Bootstrap resolves on the connecting channel's own event loop. + Bootstrap baseBootstrap; + EventLoop eventLoop; try { - resolvedAddress = endPoint.resolve(); + baseBootstrap = newBootstrap(); + eventLoop = context.getNettyOptions().ioEventLoopGroup().next(); } catch (Exception e) { resultFuture.completeExceptionally(e); return; } - NettyOptions nettyOptions = context.getNettyOptions(); + // EndPoint.resolve() is contractually non-blocking and performs no name resolution, so it is + // safe to call here even though connect() runs on the admin event loop for control-connection + // reconnects. Everything a name needs to become connectable happens in resolveCandidates(). + SocketAddress address; + try { + address = endPoint.resolve(); + } catch (Exception e) { + resultFuture.completeExceptionally(e); + return; + } + if (address == null) { + // EndPoint.resolve() is contractually non-null; fail fast instead of NPE-ing inside an + // event-loop task later, which would leave resultFuture hanging (see resolveCandidates()). + resultFuture.completeExceptionally( + new IllegalArgumentException("EndPoint.resolve() returned null: " + endPoint)); + return; + } + resolveCandidates( + baseBootstrap, address, eventLoop, spreadAcrossAddresses(endPoint, nodeIsIdentified)) + .whenComplete( + (candidates, error) -> { + if (error != null) { + Throwable cause = + (error instanceof CompletionException && error.getCause() != null) + ? error.getCause() + : error; + resultFuture.completeExceptionally(cause); + return; + } + tryNextCandidate( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + nodeIsIdentified, + resultFuture, + candidates, + 0, + new ArrayList<>()); + }); + } + + /** + * Builds the {@link Bootstrap} shared by every connection attempt of a single {@code connect()} + * call, including the user's {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} hook. Per + * attempt, {@link #connectToAddress} takes a {@link + * Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it bound to the event loop the connect + * picked, and installs its own handler; the copy carries the resolver configuration over. The + * base bootstrap itself keeps the full I/O group, so the hook observes the same group as always. + */ + private Bootstrap newBootstrap() { + NettyOptions nettyOptions = context.getNettyOptions(); Bootstrap bootstrap = new Bootstrap() .group(nettyOptions.ioEventLoopGroup()) .channel(nettyOptions.channelClass()) - .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()) - .handler( - initializer(endPoint, currentVersion, options, nodeMetricUpdater, resultFuture)); - + .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()); nettyOptions.afterBootstrapInitialized(bootstrap); + if (bootstrap.config().handler() != null && loggedHandlerWarning.compareAndSet(false, true)) { + LOG.warn( + "[{}] NettyOptions.afterBootstrapInitialized() installed a channel handler on the" + + " bootstrap; it will be replaced by the driver's own handler. Use" + + " NettyOptions.afterChannelInitialized() to customize the pipeline instead.", + logPrefix); + } + return bootstrap; + } + + /** + * Turns the address an {@link EndPoint} denotes into the concrete, connectable addresses to try, + * expanding it to all the addresses it maps to when it is a name. + * + *

Expansion goes through the bootstrap's Netty {@link AddressResolverGroup} rather than a + * direct {@code InetAddress.getAllByName()} call, so a custom resolver installed via {@link + * NettyOptions#afterBootstrapInitialized(Bootstrap)} is honoured — that is the resolver an + * unresolved address would have reached had it been handed straight to {@code + * Bootstrap.connect()}, as it was before multi-address support. This is also why endpoints are + * forbidden from resolving names themselves (see {@link EndPoint#resolve()}): doing it here is + * the only way to keep that configuration point working, and the only way to keep {@code + * resolve()} non-blocking. + * + *

Whether an address needs resolving at all is the resolver's decision, not ours: exactly as + * in {@code Bootstrap#doResolveAndConnect0}, the address is passed through untouched only when + * the resolver says it does not {@linkplain AddressResolver#isSupported support} it (e.g. {@link + * io.netty.channel.local.LocalAddress}) or that it {@linkplain AddressResolver#isResolved is + * already resolved}. Both are overridable, and a custom resolver may well report an + * already-resolved address as unresolved in order to redirect it — Netty consulted it either way, + * so a pre-check here on {@code InetSocketAddress#isUnresolved()} would silently take that + * configuration point away for every connect to an already-resolved node, which is to say for + * almost every connect. A null group means the user called {@link Bootstrap#disableResolver()}, + * which is likewise respected. + * + *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, + * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is + * the pre-existing behaviour of handing an unresolved address to {@code Bootstrap.connect()}, and + * it is an I/O loop, never the admin loop that {@code connect()} is called from. Deployments that + * need non-blocking resolution can now install {@code DnsAddressResolverGroup} and have it take + * effect. + */ + private CompletionStage> resolveCandidates( + Bootstrap bootstrap, + SocketAddress address, + EventLoop eventLoop, + boolean spreadAcrossAddresses) { + + AddressResolverGroup resolverGroup = bootstrap.config().resolver(); + if (resolverGroup == null) { + // Bootstrap.disableResolver(): the user wants the address passed through as-is, which only + // works if it is usable as-is. + IllegalStateException unusable = + unusableWithoutResolution(address, "the bootstrap has name resolution disabled"); + return (unusable != null) + ? CompletableFutures.failedFuture(unusable) + : CompletableFuture.completedFuture(Collections.singletonList(address)); + } - ChannelFuture connectFuture; - if (shardId == null || shardingInfo == null) { - if (shardId != null) { + // The supplied event loop is the same one the channel will be registered on (see connect()), + // which is what Netty itself does with an unresolved address: Bootstrap resolves on the + // connecting channel's own event loop. Its transport also matches the channel class, which + // matters because DnsAddressResolverGroup registers a datagram channel on the executor it + // resolves for. + CompletableFuture> result = new CompletableFuture<>(); + // Every path below must complete `result`: nothing at this stage has a timeout, so a task or + // listener that dies with the future still pending (Netty swallows their throwables, it only + // logs them) would hang the connect attempt -- and with it control-connection init or a pool + // reconnect -- forever. Hence the blanket catches around the task body, the listener body, and + // the execute() call itself (which throws RejectedExecutionException while shutting down). + try { + eventLoop.execute( + () -> { + try { + AddressResolver resolver = + resolverGroup.getResolver(eventLoop); + if (!resolver.isSupported(address) || resolver.isResolved(address)) { + // Nothing for the resolver to do; same short-circuit as + // Bootstrap#doResolveAndConnect0. An address the resolver declines is in the same + // position as one with no resolver at all, so it gets the same check; an + // already-resolved one is usable by definition and passes straight through. + IllegalStateException unusable = + unusableWithoutResolution( + address, "the configured resolver does not support this address"); + if (unusable != null) { + result.completeExceptionally(unusable); + } else { + result.complete(Collections.singletonList(address)); + } + return; + } + resolver + .resolveAll(address) + .addListener( + (Future> future) -> { + try { + if (!future.isSuccess()) { + result.completeExceptionally(future.cause()); + return; + } + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + if (addresses == null || addresses.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + "Resolver returned no address for " + address)); + return; + } + List connectable = + dropUnresolved(address, reattachHostnames(address, addresses)); + if (connectable.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + String.format( + "Cannot connect to %s: the configured resolver (%s) " + + "expanded it to %d address(es) and every one of them " + + "is still unresolved, so nothing will resolve them.", + address, resolver.getClass().getName(), addresses.size()))); + return; + } + result.complete(shuffleAndLimit(connectable, spreadAcrossAddresses)); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + return result; + } + + /** + * The failure to report when {@link #resolveCandidates} is about to pass an address through + * without resolving it, or {@code null} if passing it through is fine. + * + *

{@link #connectToAddress} hands the candidate to a bootstrap clone with {@link + * Bootstrap#disableResolver()}, so an address that is still unresolved by the time it gets there + * cannot connect: Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}, + * naming neither the address nor the reason nothing resolved it. That is a hard failure of every + * connection attempt for the whole session, and it is worth a message that says which endpoint + * and which configuration produced it -- the endpoints most likely to hit it (SNI, client routes) + * hand out unresolved addresses by design, and contact-point hostnames are now always kept + * unresolved. + * + *

Deliberately not a general {@code isUnresolved()} pre-check on every path: see {@link + * #resolveCandidates}'s javadoc for why an address the resolver merely declines to touch must + * still go through. This fires only where nothing downstream will resolve it either -- as does + * {@link #dropUnresolved}, which applies the same reasoning to what {@code resolveAll} returns. + */ + private static IllegalStateException unusableWithoutResolution( + SocketAddress address, String why) { + if (!(address instanceof InetSocketAddress) || !((InetSocketAddress) address).isUnresolved()) { + return null; + } + return new IllegalStateException( + String.format( + "Cannot connect to %s: it is an unresolved address and %s, so nothing will resolve it. " + + "Either remove Bootstrap.disableResolver() from " + + "NettyOptions.afterBootstrapInitialized(), or supply an already-resolved address.", + address, why)); + } + + /** Applies {@link #reattachHostname} to every expanded candidate. */ + private static List reattachHostnames( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + result.add(reattachHostname(original, candidate)); + } + return result; + } + + /** + * Drops the candidates a resolver returned still unresolved, keeping the order of the rest. + * + *

{@code resolveAll} is contracted to return resolved addresses, but a custom resolver that + * rewrites what it is given -- which {@link #resolveCandidates} deliberately supports -- may hand + * back one that is not. Such a candidate cannot connect: {@link #connectToAddress} uses a + * bootstrap clone with {@link Bootstrap#disableResolver()}, so nothing downstream will resolve it + * either, and Netty raises {@code UnresolvedAddressException} from inside {@code doConnect}. This + * is the same reasoning as {@link #unusableWithoutResolution}, applied where the addresses come + * from the resolver itself; dropping them here rather than after the cap means the cap counts + * only addresses that can actually be tried. The caller reports the case where nothing is left, + * which is the one that fails every connection attempt for the whole session. + */ + private List dropUnresolved( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + if (candidate instanceof InetSocketAddress + && ((InetSocketAddress) candidate).isUnresolved()) { LOG.debug( - "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", - shardId, - endPoint); - } - connectFuture = bootstrap.connect(resolvedAddress); - } else { - int localPort = - PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); - if (localPort == -1) { - LOG.warn( - "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", - shardId, - endPoint); - connectFuture = bootstrap.connect(resolvedAddress); + "[{}] Resolver returned {} for {} but it is still unresolved, skipping it", + logPrefix, + candidate, + original); } else { - connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + result.add(candidate); } } + return result; + } - connectFuture.addListener( - cf -> { - if (connectFuture.isSuccess()) { - Channel channel = connectFuture.channel(); - DriverChannel driverChannel = - new DriverChannel(endPoint, channel, context.getWriteCoalescer(), currentVersion); - // If this is the first successful connection, remember the protocol version and - // cluster name for future connections. - if (isNegotiating) { - ChannelFactory.this.protocolVersion = currentVersion; - } - if (ChannelFactory.this.clusterName == null) { - ChannelFactory.this.clusterName = driverChannel.getClusterName(); - } - Map> supportedOptions = driverChannel.getOptions(); - if (ChannelFactory.this.productType == null && supportedOptions != null) { - List productTypes = supportedOptions.get("PRODUCT_TYPE"); - String productType = - productTypes != null && !productTypes.isEmpty() - ? productTypes.get(0) - : UNKNOWN_PRODUCT_TYPE; - ChannelFactory.this.productType = productType; - DriverConfig driverConfig = context.getConfig(); - if (driverConfig instanceof TypesafeDriverConfig - && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { - ((TypesafeDriverConfig) driverConfig) - .overrideDefaults( - ImmutableMap.of( - DefaultDriverOption.REQUEST_CONSISTENCY, - ConsistencyLevel.LOCAL_QUORUM.name())); - } - } - resultFuture.complete(driverChannel); - } else { - Throwable error = connectFuture.cause(); - if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { - attemptedVersions.add(currentVersion); - Optional downgraded = - context.getProtocolVersionRegistry().downgrade(currentVersion); - if (downgraded.isPresent()) { + /** + * Re-attaches the {@code original} address's host name to one of the resolved candidates it + * expanded to, whatever name that candidate carries. + * + *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es + * they return, so this is a no-op for them. A custom resolver, however, may build its results + * from raw address bytes, or label them with a canonical/CNAME name of its own. The channel's + * pinned endpoint is built from the candidate (see {@link PinnableEndPoint}), and it is what + * {@code DefaultSslEngineFactory} and {@code SniSslEngineFactory} derive the SSL peer host from, + * inside the channel initializer. So whatever name the candidate carries is the name TLS hostname + * verification checks the server certificate against, and the only name that may be is the one + * the user configured: with a nameless address, {@code InetSocketAddress#getHostName()} + * additionally triggers a blocking reverse-DNS lookup on the event loop and validation falls back + * to the IP or the PTR record, and with a resolver-supplied label it validates a name the + * operator never chose. Hence the queried name always wins here; before multi-address support the + * initializer kept the original endpoint and Netty resolved only the TCP destination, which had + * the same effect. + * + *

Re-attaching changes nothing else: {@code InetAddress.getByAddress(host, bytes)} performs no + * lookup, the TCP connect target is the same IP, and a resolved {@link InetSocketAddress}'s + * equality ignores host names, so pinning and the pin-equality shortcuts are unaffected. A scoped + * IPv6 candidate keeps its scope, since {@link Inet6Address} has {@code getByAddress} overloads + * that carry one. + * + *

An original that carries no name of its own is left alone (see {@link + * AddressUtils#carriesName}): a resolver is free to redirect it to a different IP, and labelling + * that IP with the literal form of the one we asked for would invent a name that resolves to + * something else. + */ + @VisibleForTesting + static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { + if (!(original instanceof InetSocketAddress) || !(candidate instanceof InetSocketAddress)) { + return candidate; + } + InetSocketAddress originalInet = (InetSocketAddress) original; + InetSocketAddress candidateInet = (InetSocketAddress) candidate; + InetAddress candidateIp = candidateInet.getAddress(); + if (!AddressUtils.carriesName(originalInet) + || candidateIp == null + // Nothing to change: the candidate already carries the queried name, which is the common + // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks + // anything up -- for a nameless address it falls back to the IP literal. + || candidateInet.getHostString().equals(originalInet.getHostString())) { + return candidate; + } + try { + return new InetSocketAddress( + withHostName(originalInet.getHostString(), candidateIp), candidateInet.getPort()); + } catch (UnknownHostException impossible) { + // getByAddress only rejects illegal byte lengths, and these bytes come from a real + // InetAddress; keep the raw candidate rather than failing the connect over a cosmetic step. + return candidate; + } + } + + /** + * Returns a copy of {@code ip} labelled with {@code hostName}, preserving an IPv6 scope if there + * is one. + * + *

{@link InetAddress#getByAddress(String, byte[])} cannot carry a scope, and dropping one + * would change where the address actually points — a link-local address is only meaningful + * together with its zone. {@link Inet6Address#getByAddress(String, byte[], int)} carries the zone + * as its numeric id, which is what the connect itself goes on; a scope id of 0 means "unscoped" + * and is accepted, so this needs no special case for a plain IPv6 address. + * + *

The sibling overload taking a {@link NetworkInterface} is deliberately not used: it + * re-derives the numeric scope by searching that interface for an address of the same local type, + * and throws {@code UnknownHostException("no scope_id found")} when it finds none — so it can + * fail for an address that was legitimately built from an interface in the first place. All that + * is lost by going numeric is the interface name, which surfaces in {@code toString()} and + * nowhere else. + */ + private static InetAddress withHostName(String hostName, InetAddress ip) + throws UnknownHostException { + return ip instanceof Inet6Address + ? Inet6Address.getByAddress(hostName, ip.getAddress(), ((Inet6Address) ip).getScopeId()) + : InetAddress.getByAddress(hostName, ip.getAddress()); + } + + /** + * Whether {@link #shuffleAndLimit} may spread this connect across the addresses the endpoint + * expands to. + * + *

A contact point always may: its addresses may well be different nodes, so there is no + * node identity to preserve, and spreading both balances load and varies which address an attempt + * starts from. For an {@linkplain #isIdentified(Node) identified} node it depends on what the + * name denotes, which only the endpoint knows — see {@link + * PinnableEndPoint#addressesAreInterchangeable()} for the two cases and why they differ. An + * endpoint that does not implement {@link PinnableEndPoint} is treated as not interchangeable, + * which is also the conservative reading for a third-party implementation. + */ + @VisibleForTesting + static boolean spreadAcrossAddresses(EndPoint endPoint, boolean nodeIsIdentified) { + return !nodeIsIdentified + || (endPoint instanceof PinnableEndPoint + && ((PinnableEndPoint) endPoint).addressesAreInterchangeable()); + } + + /** + * Truncates the expanded address list to {@code advanced.connection.max-candidate-addresses}, + * shuffling it first when the addresses may be spread across (see {@link + * #spreadAcrossAddresses}). + * + *

The shuffle spreads load: without it, every connection would try the resolver's first + * address first and healthy connections would pile onto one IP, while the whole point of a + * multi-record name is usually to spread them. A fresh random order per connect also means + * successive attempts start at different addresses, with no per-name counter state to maintain + * and nothing depending on the order the resolver, or a sort, happened to choose. + * + *

Where the order is kept instead, it is because the addresses are not known to be + * interchangeable: an identified node whose endpoint is an unresolved name that may map to + * several hosts, which is what a configured {@code AddressTranslator} returns by default ({@code + * SubnetAddressTranslator} under {@code resolve-addresses = false}). Each pool connection is its + * own {@code connect()}, so shuffling there would land one {@code Node}'s channels on different + * hosts while routing, shard awareness and per-node metrics attribute them all to that node. + * Keeping the resolver's order means such a pool converges on one address, as it did before + * multi-address support -- {@code Bootstrap.connect()} resolved through {@code resolve()}, + * singular, i.e. the first record -- while the remaining addresses still serve as fallback. + * + *

The cap bounds what a single connect attempt can cost: every address tried is a full TCP + * connect plus init handshake -- and, with wrong credentials, a rejected login (see {@link + * #tryNextCandidate} on why an authentication failure does not stop the loop). For a shuffled + * list, a capped attempt tries a different sample of the addresses each time, so a name with more + * records than the cap still reaches all of them across successive attempts; one attempt just no + * longer walks them all. Where the order is kept, the cap is a hard limit -- a capped attempt + * keeps dialing the same prefix of the list, so records beyond it are never reached. That is + * accepted rather than worked around: it is still strictly more than the single address such an + * endpoint got before multi-address support, and rotating the window instead would give up the + * convergence the stable order exists for. + */ + @VisibleForTesting + List shuffleAndLimit( + List addresses, boolean spreadAcrossAddresses) { + List shuffled = new ArrayList<>(addresses); + if (shuffled.size() > 1 && spreadAcrossAddresses) { + Collections.shuffle(shuffled, random); + } + int cap = + Math.max( + 1, + context + .getConfig() + .getDefaultProfile() + .getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)); + if (shuffled.size() > cap) { + LOG.debug( + "[{}] Resolved {} addresses, will try at most {}" + + " (advanced.connection.max-candidate-addresses)", + logPrefix, + shuffled.size(), + cap); + return shuffled.subList(0, cap); + } + return shuffled; + } + + /** + * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one + * in sequence; when an address fails, the next candidate is tried, and only when all candidates + * are exhausted is the overall {@code resultFuture} failed. + * + *

One failure is node-wide -- it dooms every remaining address rather than only the one + * that was tried: an {@link UnsupportedProtocolVersionException} against an identified + * node ({@code nodeIsIdentified}, see {@link #isIdentified(Node)}). Every address of an + * identified node is that same node, so a protocol-version rejection is a property of all of + * them, and the attempt fails immediately instead of replaying the negotiation against every + * remaining IP. This matches the pre-multi-address behaviour of a single-address connect. The + * corner it deliberately does not rescue: a heterogeneous rolling upgrade where different IPs of + * one identified node genuinely support different protocol versions. + * + *

Note what that leaves uncovered, because it is narrower than it looks: {@code isNegotiating} + * is true only while {@link #protocolVersion} is still unset, i.e. on the session's first + * connection, which is always to a contact point -- and a contact point is never identified (see + * {@link #isIdentified}). So a rejection reached by negotiation never takes this branch; + * only a forced version rejected by an already-identified node does, which is the + * single-attempt case. A negotiated rejection still walks the downgrade ladder from the top on + * every candidate, since each gets a fresh {@code attemptedVersions} list, so a name with N + * records costs N ladders (with N bounded by {@link #shuffleAndLimit}). + * + *

Every other failure -- authentication included -- advances to the next candidate: the + * addresses a name expands to may well belong to different nodes, so a rejection by the first of + * them says nothing about the rest. That also preserves the behaviour this PR would otherwise + * have removed: with {@code advanced.resolve-contact-points = true} each resolved address used to + * be a separate {@code Node}, and {@code ControlConnection} advances to the next node in its + * query plan on any error, including these. + * + *

Authentication in particular has to advance, for a reason only visible in the order of the + * handshake: {@link ProtocolInitHandler} runs {@code STARTUP -> AUTH_RESPONSE -> + * GET_CLUSTER_NAME}, so authentication completes before the cluster-name check. A stale + * DNS record pointing at a foreign cluster that wants different credentials therefore fails at + * AUTH, and treating that as terminal would write off the whole hostname -- making the + * cluster-name mismatch that would have advanced to the next address unreachable, in exactly the + * multi-record case this loop exists for. What bounds the cost of genuinely wrong credentials is + * the candidate cap ({@link #shuffleAndLimit}): one attempt pays at most {@code + * advanced.connection.max-candidate-addresses} rejected logins, with the earlier failures + * attached as suppressed exceptions. + * + *

Timeout note: addresses are tried serially, so the worst-case time before failure is + * N times a full attempt, and an attempt is a connect plus the init handshake. Each of the + * handshake's steps arms its own {@code advanced.connection.init-query-timeout} when it is sent, + * so they accumulate instead of sharing one deadline: a single address that accepts the + * connection and then stalls costs {@code connect-timeout} plus several times {@code + * init-query-timeout} before the loop moves on. This is an intentional tradeoff: failing + * immediately on the first unreachable IP would prevent fallback to healthy ones. The candidate + * cap ({@link #shuffleAndLimit}) is what bounds N. + * + *

When every candidate fails, one of their errors is propagated -- see {@link + * #surfacedFailure} for which, and why it is not simply the last -- with every other candidate's + * failure attached to it as a {@linkplain Throwable#addSuppressed(Throwable) suppressed} + * exception, so no cause is lost. + * + *

What that does not give is which address produced which failure. Every one of these + * exceptions is built from the endpoint, and a pinned copy is required to render identically to + * the unpinned original ({@link PinnableEndPoint}), so a three-record name yields three messages + * that all name the same hostname. The DEBUG line above is where the pairing lives; making the + * exceptions themselves carry it would mean either relaxing that contract or wrapping causes in a + * driver-owned type, which would in turn break the {@code instanceof} tests the callers do on + * them (see {@link #surfacedFailure}). + */ + private void tryNextCandidate( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + boolean nodeIsIdentified, + CompletableFuture resultFuture, + List candidates, + int index, + List priorErrors) { + + // Invariant: this method always (eventually) completes resultFuture. It is invoked from + // CompletionStage and Netty callbacks that swallow throwables, so a synchronous throw -- a + // custom PinnableEndPoint.pinTo() for instance -- would otherwise leave the connect attempt + // hanging forever. Double completion is harmless: completeExceptionally() on an already + // completed future is a no-op. + try { + SocketAddress candidate = candidates.get(index); + // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and + // the DriverChannel handed to the caller -- sees an endpoint bound to this one address + // instead of the multi-address original. See PinnableEndPoint for why that matters. + EndPoint pinnedEndPoint = pin(endPoint, candidate); + CompletableFuture perAddressFuture = new CompletableFuture<>(); + // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one + // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms + // this candidate) only reports versions actually tried against it, not earlier candidates'. + List attemptedVersions = new CopyOnWriteArrayList<>(); + connectToAddress( + baseBootstrap, + eventLoop, + pinnedEndPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + candidate); + + perAddressFuture.whenComplete( + (channel, error) -> { + try { + boolean nodeWide = error != null && isNodeWideFailure(error, nodeIsIdentified); + if (error == null) { + if (!resultFuture.complete(channel)) { + // Same guard as completeCandidate and abandonCandidate: resultFuture is handed to + // callers as a CompletionStage and every path that can complete it early does so + // exceptionally (the blanket catches in resolveCandidates and below), so losing + // this race is possible -- and would otherwise leak a live socket and its + // pipeline for the life of the JVM, since nobody else holds this channel. + channel.forceClose(); + } + } else if (!nodeWide && index + 1 < candidates.size()) { LOG.debug( - "[{}] Failed to connect with protocol {}, retrying with {}", + "[{}] Failed to connect to {} ({}), trying next address", logPrefix, - currentVersion, - downgraded.get()); - connect( + candidate, + error.getMessage()); + priorErrors.add(error); + tryNextCandidate( + baseBootstrap, + eventLoop, + // Deliberately the original, not the pinned copy: the next candidate must be + // pinned from the unpinned endpoint. endPoint, shardingInfo, shardId, options, nodeMetricUpdater, - downgraded.get(), - true, - attemptedVersions, - resultFuture); + currentVersion, + isNegotiating, + nodeIsIdentified, + resultFuture, + candidates, + index + 1, + priorErrors); } else { - resultFuture.completeExceptionally( - UnsupportedProtocolVersionException.forNegotiation( - endPoint, attemptedVersions)); + if (index + 1 < candidates.size()) { + // Only reachable for a node-wide failure (see the javadoc). + LOG.debug( + "[{}] Not trying the remaining addresses of {}: this failure is a property of" + + " the node, not of the address ({})", + logPrefix, + endPoint, + error.getMessage()); + } + // Surface one failure, carrying the others as suppressed exceptions so they are + // not lost (they were only logged at DEBUG above). Deduplicated by identity: + // nothing stops two candidates from failing with the same Throwable instance, and + // this mutates an object we do not own -- attaching it twice would show the same + // cause twice, and would keep growing a shared instance's suppressed list on every + // connect. + List allErrors = new ArrayList<>(priorErrors); + allErrors.add(error); + Throwable surfaced = surfacedFailure(allErrors, nodeWide); + Set attached = Collections.newSetFromMap(new IdentityHashMap<>()); + attached.add(surfaced); + for (Throwable candidateError : allErrors) { + if (attached.add(candidateError)) { + surfaced.addSuppressed(candidateError); + } + } + // Note: might be completed already if the failure happened in initializer() + resultFuture.completeExceptionally(surfaced); } + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + }); + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + } + + /** + * Whether {@code error} dooms every remaining address of the endpoint, making it pointless for + * {@link #tryNextCandidate} to try them. See its javadoc for the reasoning behind each case. + * + *

A {@link ClusterNameMismatchException} is deliberately absent, for an identified node as + * much as for a contact point. It says that the address just tried fronts a different cluster, + * which is a property of that record rather than of the node -- a stale DNS entry is exactly what + * it looks like -- so advancing to the next address is the whole point. {@link #surfacedFailure} + * treats it with the same caution on the way out. + * + *

An {@link AuthenticationException} is deliberately absent too, even for an identified node: + * see {@link #tryNextCandidate} on why authentication must advance, and {@link #shuffleAndLimit} + * for the cap that bounds what wrong credentials can cost. + * + *

An {@link UnsupportedEventTypeException} is node-wide, for a contact point as much as + * for an identified node, and unlike every other case here it does not depend on {@code + * nodeIsIdentified}: it says the server does not know an event type the driver asked to register + * for, which is a property of the software running there, not of the record that reached it. + * Replaying it against the remaining addresses can only fail the same way, and each replay costs + * a full TCP connect plus the STARTUP/AUTH/cluster-name handshake and the connect hook's round + * trip. Stopping at the first one restores what this rejection cost while REGISTER was an init + * step, which was one failed connect per contact point. + */ + private static boolean isNodeWideFailure(Throwable error, boolean nodeIsIdentified) { + return (error instanceof UnsupportedProtocolVersionException && nodeIsIdentified) + || error instanceof UnsupportedEventTypeException; + } + + /** + * Which of the candidates' failures to propagate once they are all exhausted, the rest being + * attached to it as suppressed exceptions. + * + *

Not simply the last one. Callers branch on the type of what they receive -- {@link + * com.datastax.oss.driver.internal.core.pool.ChannelPool#handleError} treats a cluster-name + * mismatch and a protocol-version rejection as fatal, an invalid keyspace as a keyspace error and + * an authentication failure as warn-and-retry; {@code ControlConnection} logs authentication + * failures differently from transport ones -- and with a multi-record name the address that + * happens to be tried last is arbitrary. Letting it win would report a firewalled IP's connect + * timeout for what is really a rejected password, and take the reconnect path where the caller + * asked for the fatal one. + * + *

So a failure the callers classify is preferred over one they do not, in the order they test + * for it, and the last non-fatal failure is only used when no candidate produced a + * classified one. Every failure is still attached, so nothing is lost either way. + * + *

The two fatal types are the exception to that: they are only preferred when every + * candidate failed that way, or when the last one is the node-wide failure that stopped the loop. + * See the comments in the body. + */ + private static Throwable surfacedFailure(List errors, boolean lastIsNodeWide) { + Throwable lastError = errors.get(errors.size() - 1); + // A node-wide failure is what ended the loop, and it is a verdict about the node rather than + // about the one address it was observed on (see tryNextCandidate). It therefore outranks + // everything below, including the unanimity rule -- which would otherwise demote it to whatever + // transport failure an earlier address happened to produce, turning the forced-down node that a + // single-address connect has always produced into a reconnect. + if (lastIsNodeWide) { + return lastError; + } + // An irreversible verdict needs evidence from every address. handleError turns these two into + // TopologyEvent.forceDown, and nothing in the driver ever reverses one -- no component fires + // FORCE_UP, and a SUGGEST_UP is explicitly refused for a FORCED_DOWN node -- so the node is out + // for the rest of the session. Meanwhile tryNextCandidate() classifies a cluster-name mismatch + // as a property of the address and advances past it, which is the point: it means this record + // is stale, not that this node belongs to another cluster. Promoting one such record over the + // other candidates' transport failures would write a healthy node off on the strength of the + // one address that was never going to work. Requiring unanimity leaves a single-address + // endpoint exactly as it was before this loop existed -- one candidate is unanimous by + // definition -- and a mixed pass simply reconnects, forcing down on the first pass that is. + boolean everyCandidateFatal = true; + for (Throwable error : errors) { + if (!isFatalToCallers(error)) { + everyCandidateFatal = false; + break; + } + } + if (everyCandidateFatal) { + return errors.get(0); + } + // An invalid keyspace, unlike those two, is a property of the cluster's schema rather than of + // the address, so one address answering settles it and no unanimity is required. It outranks an + // authentication failure because it is the rung a caller acts on -- handleError routes it to + // onKeyspaceError, which is how PoolManager fails session init fast instead of reconnecting for + // a keyspace that will never appear -- and because reaching the keyspace step at all proves the + // credentials were accepted on that address. + for (Throwable error : errors) { + if (error instanceof InvalidKeyspaceException) { + return error; + } + } + for (Throwable error : errors) { + if (error instanceof AuthenticationException) { + return error; + } + } + // The last failure -- but not a fatal one. The address tried last is arbitrary, so promoting a + // fatal failure here would force the node down on the strength of the single address that + // produced it, which is exactly what the unanimity rule above refuses to do. + for (int i = errors.size() - 1; i >= 0; i--) { + Throwable error = errors.get(i); + if (!isFatalToCallers(error)) { + return error; + } + } + // Unreachable: a list of nothing but fatal failures returned at the unanimity check above. + return lastError; + } + + /** + * Whether the callers treat {@code error} as fatal, i.e. as grounds to write the node off: {@code + * ChannelPool#handleError} turns these two, and only these two, into {@code + * TopologyEvent.forceDown}. + */ + private static boolean isFatalToCallers(Throwable error) { + return error instanceof ClusterNameMismatchException + || error instanceof UnsupportedProtocolVersionException; + } + + /** + * Performs a Netty bootstrap connect to a single, already-resolved address. Handles + * protocol-version negotiation (downgrade retries) internally, staying on the same address. Uses + * {@code perAddressFuture} so {@link #tryNextCandidate} can distinguish a per-address TCP failure + * (try the next IP) from a successful protocol handshake. + */ + private void connectToAddress( + Bootstrap baseBootstrap, + EventLoop eventLoop, + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CompletableFuture perAddressFuture, + SocketAddress resolvedAddress) { + + // Invariant, as in tryNextCandidate(): every path completes perAddressFuture. The synchronous + // section can throw from Bootstrap validation; the connect listener runs inside a Netty + // callback that swallows throwables and contains the downgrade recursion, the version-registry + // lookup and the config overrides, any of which throwing would otherwise hang the attempt. + try { + // clone(eventLoop) so each attempt gets its own handler while sharing the options (including + // anything afterBootstrapInitialized() set), and is registered on the event loop the + // connect() picked -- the same one resolution ran on, so the group's chooser advances exactly + // once per logical connect (see connect()). + // + // disableResolver() because resolveCandidates() has already done the one resolution pass this + // connect gets, and `resolvedAddress` is one of its results. Bootstrap.clone() otherwise + // carries the resolver over and Netty resolves again -- through resolve(), *singular*. That + // is inert for the default resolver, which short-circuits on isResolved(), but a resolver + // that reports resolved addresses as unresolved in order to redirect them -- which + // resolveCandidates() deliberately supports -- would remap every candidate onto its first + // answer: the remaining candidates would never actually be tried, and the endpoint pinned + // onto the channel would name an address the channel is not connected to (which is what the + // SSL engine's peer host and DefaultTopologyMonitor#savePort are derived from). Every other + // exit from resolveCandidates() yields an address Netty would itself have passed through + // untouched -- no group, !isSupported, or isResolved -- so nothing else changes. + Bootstrap bootstrap = + baseBootstrap + .clone(eventLoop) + .disableResolver() + .handler( + initializer( + endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); + + ChannelFuture connectFuture; + if (shardId == null || shardingInfo == null) { + if (shardId != null) { + LOG.debug( + "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", + shardId, + endPoint); + } + connectFuture = bootstrap.connect(resolvedAddress); + } else { + int localPort = + PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); + if (localPort == -1) { + LOG.warn( + "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", + shardId, + endPoint); + connectFuture = bootstrap.connect(resolvedAddress); + } else { + connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + } + } + + connectFuture.addListener( + cf -> { + try { + if (connectFuture.isSuccess()) { + Channel channel = connectFuture.channel(); + DriverChannel driverChannel = + new DriverChannel( + endPoint, channel, context.getWriteCoalescer(), currentVersion); + finishCandidate( + driverChannel, + options, + perAddressFuture, + () -> latchNegotiatedState(driverChannel, currentVersion, isNegotiating)); + } else { + Throwable error = connectFuture.cause(); + if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { + attemptedVersions.add(currentVersion); + Optional downgraded = + context.getProtocolVersionRegistry().downgrade(currentVersion); + if (downgraded.isPresent()) { + LOG.debug( + "[{}] Failed to connect with protocol {}, retrying with {}", + logPrefix, + currentVersion, + downgraded.get()); + // Stay on the same address for protocol-version downgrade retries. + connectToAddress( + baseBootstrap, + eventLoop, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + downgraded.get(), + true, + attemptedVersions, + perAddressFuture, + resolvedAddress); + } else { + perAddressFuture.completeExceptionally( + UnsupportedProtocolVersionException.forNegotiation( + endPoint, attemptedVersions)); + } + } else { + // Note: might be completed already if the failure happened in initializer(), this + // is fine + perAddressFuture.completeExceptionally(error); + } + } + } catch (Throwable t) { + // Close the channel we opened before giving up on it. Nothing else holds it once this + // listener returns -- the DriverChannel wrapper is out of scope, and no candidate was + // completed with it -- so the socket and its pipeline would stay open for the life of + // the JVM. Only when the future is ours to fail, though: a candidate that completed + // successfully is the caller's channel, not ours to close. + if ((perAddressFuture.completeExceptionally(t) + || perAddressFuture.isCompletedExceptionally()) + && connectFuture.isSuccess()) { + connectFuture.channel().close(); + } + } + }); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); + } + } + + /** + * Remembers what the first accepted connection negotiated, so later connections skip the + * negotiation: the protocol version, the cluster name to check others against, and the server's + * product type (which for Cloud also lowers the default consistency level). + * + *

Run only once a candidate has been accepted, not as soon as its transport connect and + * init handshake succeed. Init is no longer the last word on a candidate: the connect hook can + * reject it (the control connection's identity read does, for a node with no {@code host_id}), + * and REGISTER, which used to be the final init step, now runs after that hook. A candidate the + * driver is about to throw away must not leave its cluster name latched here -- a stale DNS + * record pointing at a foreign cluster would otherwise make every subsequent connection fail its + * cluster-name check, which {@code ChannelPool} turns into an irreversible forced-down node. + */ + private void latchNegotiatedState( + DriverChannel driverChannel, ProtocolVersion currentVersion, boolean isNegotiating) { + if (isNegotiating) { + this.protocolVersion = currentVersion; + } + if (this.clusterName == null) { + this.clusterName = driverChannel.getClusterName(); + } + Map> supportedOptions = driverChannel.getOptions(); + if (this.productType == null && supportedOptions != null) { + List productTypes = supportedOptions.get("PRODUCT_TYPE"); + String productType = + productTypes != null && !productTypes.isEmpty() + ? productTypes.get(0) + : UNKNOWN_PRODUCT_TYPE; + this.productType = productType; + DriverConfig driverConfig = context.getConfig(); + if (driverConfig instanceof TypesafeDriverConfig + && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { + ((TypesafeDriverConfig) driverConfig) + .overrideDefaults( + ImmutableMap.of( + DefaultDriverOption.REQUEST_CONSISTENCY, ConsistencyLevel.LOCAL_QUORUM.name())); + } + } + } + + /** + * The tail of a candidate attempt, once transport connect and protocol initialization have both + * succeeded: runs the caller's {@link ConnectHook} (if any), then registers for protocol events + * (if requested), and only then completes the candidate's future. + * + *

Both steps happen while this attempt still holds the endpoint's remaining addresses, so a + * failure in either is a per-candidate failure: the channel is force-closed on the spot and + * {@link #tryNextCandidate} advances to the next address. + * + *

REGISTER used to be the last protocol-init step; it moved behind the hook so that a channel + * the hook is about to reject never registers for events. The window in which a live channel is + * not yet registered grows by the hook's round trip -- the same order of cost as the init step it + * follows. + */ + private void finishCandidate( + DriverChannel driverChannel, + DriverChannelOptions options, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (options.connectHook == null) { + registerForEvents(driverChannel, options, perAddressFuture, onAccepted); + return; + } + CompletionStage vetted; + try { + vetted = options.connectHook.onConnect(driverChannel); + } catch (Throwable t) { + // A synchronous throw is a rejection, like an exceptional stage. Blanket-caught: this runs + // inside a Netty listener that swallows throwables, so a caller-supplied callback leaking + // one would otherwise leave the attempt hanging forever. + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", t)); + return; + } + if (vetted == null) { + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook returned a null stage", null)); + return; + } + // The hook's contract says its stage eventually completes, but only the driver can make that + // true: a wedged hook (a topology monitor whose own timeout is broken, say) would otherwise + // hang the whole connect attempt, and with it control-connection init or a reconnect. + // + // Unless the timeout is zero or negative, which every other consumer of a driver timeout option + // reads as "no timeout" (see AdminRequestHandler#onWriteComplete). Scheduling it anyway would + // fire on the next event-loop turn, before any round trip can complete, and abandon every + // candidate of every contact point -- so an operator who disabled the control-connection + // timeout + // would find that the session cannot initialize at all. + ScheduledFuture hookTimeout; + try { + hookTimeout = + (options.connectHookTimeout == null || options.connectHookTimeout.toNanos() <= 0) + ? null + : driverChannel + .eventLoop() + .schedule( + () -> + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Connect hook timed out after " + options.connectHookTimeout, + null)), + options.connectHookTimeout.toNanos(), + TimeUnit.NANOSECONDS); + } catch (Throwable t) { + // An event loop shutting down rejects the task. Fail the candidate rather than run the hook + // with nothing bounding it: this method is called from a Netty listener that swallows + // throwables, so the attempt would otherwise hang (see connectToAddress's invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Could not schedule the connect hook timeout", t)); + return; + } + vetted.whenComplete( + (aVoid, error) -> { + try { + if (hookTimeout != null) { + hookTimeout.cancel(false); + } + if (error != null) { + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Connect hook rejected the channel", error)); } else { - // Note: might be completed already if the failure happened in initializer(), this is - // fine - resultFuture.completeExceptionally(error); + registerForEvents(driverChannel, options, perAddressFuture, onAccepted); } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and the timeout that would have failed the candidate has just been + // cancelled, so anything escaping here -- registerForEvents' config read, for instance + // -- would leave perAddressFuture uncompleted forever. + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException( + "Unexpected error after the connect hook accepted the channel", t)); } }); } + /** + * Sends the REGISTER request when the options ask for protocol events, then completes the + * candidate. A registration failure is a per-candidate failure, exactly as it was when REGISTER + * was a protocol-init step. + */ + private void registerForEvents( + DriverChannel driverChannel, + DriverChannelOptions options, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (options.eventTypes.isEmpty()) { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + return; + } + Duration timeout = + context + .getConfig() + .getDefaultProfile() + .getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT); + AdminRequestHandler.register(driverChannel, options.eventTypes, timeout, logPrefix) + .start() + .whenComplete( + (aVoid, error) -> { + try { + if (error != null) { + abandonCandidate( + driverChannel, perAddressFuture, translateRegisterFailure(error)); + } else { + completeCandidate(driverChannel, perAddressFuture, onAccepted); + } + } catch (Throwable t) { + // Blanket-caught, as everywhere else in this class: nobody consumes the stage this + // callback returns, and by this point no timeout is left to fail the candidate, so + // anything escaping -- translateRegisterFailure's casts, a forceClose() on an event + // loop that is shutting down -- would leave perAddressFuture uncompleted forever + // and hang the connect attempt (see connectToAddress's invariant). + abandonCandidate( + driverChannel, + perAddressFuture, + new ConnectionInitException("Unexpected error after REGISTER", t)); + } + }); + } + + /** + * Gives the one REGISTER rejection with a known cause a message that names it: the server not + * knowing the {@code CLIENT_ROUTES_CHANGE} event type. This translation lived in the init handler + * when REGISTER was an init step, and exists so that the caller + * (ClientRoutesTopologyMonitor.init()) reports a clear error instead of silently degrading. + */ + private static Throwable translateRegisterFailure(Throwable error) { + if (error instanceof UnexpectedResponseException) { + Message response = ((UnexpectedResponseException) error).message; + if (response instanceof com.datastax.oss.protocol.internal.response.Error) { + com.datastax.oss.protocol.internal.response.Error protocolError = + (com.datastax.oss.protocol.internal.response.Error) response; + if (protocolError.code == ProtocolConstants.ErrorCode.PROTOCOL_ERROR + && protocolError.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { + return new UnsupportedEventTypeException( + "Server does not support CLIENT_ROUTES_CHANGE event " + + "(requires ScyllaDB Enterprise >= 2026.1). " + + "Either upgrade the server or remove the client routes configuration.", + error); + } + } + } + return error; + } + + /** + * A REGISTER rejection that is a property of the server -- it does not know an event type + * the driver asked for -- rather than of the address that was dialled. + * + *

A {@link ConnectionInitException}, so that callers which branch on the type (see {@link + * #surfacedFailure}, and {@code ClientRoutesTopologyMonitor#init}, which reports the message) + * treat it exactly as they treated the same rejection when REGISTER was an init step. The subtype + * exists only so {@link #isNodeWideFailure} can recognise it. + */ + @VisibleForTesting + static class UnsupportedEventTypeException extends ConnectionInitException { + UnsupportedEventTypeException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * Completes the candidate's future, closing the channel if something else -- the hook timeout, + * the only concurrent completer -- got there first: nothing else would close it, and an + * unpublished channel nobody holds is a leak. + * + *

Winning that race is also what makes this candidate the accepted one, and therefore the only + * one entitled to record what it negotiated (see {@link #latchNegotiatedState}). + */ + private static void completeCandidate( + DriverChannel driverChannel, + CompletableFuture perAddressFuture, + Runnable onAccepted) { + if (perAddressFuture.complete(driverChannel)) { + onAccepted.run(); + } else { + driverChannel.forceClose(); + } + } + + /** + * Closes a candidate channel that will not be used and fails its future -- unless that channel + * has meanwhile been handed to the caller, in which case it is theirs and must be left alone. + */ + private static void abandonCandidate( + DriverChannel driverChannel, + CompletableFuture perAddressFuture, + Throwable error) { + // The hook timeout and the hook's own completion race, and the hook's stage may complete off + // the + // channel's event loop -- the contract allows it, and a custom TopologyMonitor behind the + // control + // connection's hook is free to -- so cancel(false) can lose to a timeout task that has already + // started running. The loser of that race must not close a channel the winner published: + // completeExceptionally() is a silent no-op on an already *successful* future, so closing + // unconditionally would leave the caller owning a dead channel and no error to explain it. + // Failing it ourselves, or finding it already failed (by another abandonCandidate, or by the + // connect listener), is what makes the channel ours to close; forceClose is idempotent. + if (perAddressFuture.completeExceptionally(error) + || perAddressFuture.isCompletedExceptionally()) { + driverChannel.forceClose(); + } + } + + /** + * Binds {@code endPoint} to the address a connection is being opened to, when the implementation + * supports it. + * + *

Third-party {@link EndPoint}s that do not implement {@link PinnableEndPoint} are returned + * unchanged, so they keep behaving exactly as they did before multi-address support: the channel + * carries the endpoint it was given. + * + *

So is an endpoint whose candidate came back unresolved. {@link + * PinnableEndPoint#pinTo(SocketAddress)} is documented to take an address that is already + * resolved, and {@link #resolveCandidates} has three paths that hand the original address + * straight through -- the user disabled the resolver, the resolver does not support the address, + * or it reports it as already resolved. For an endpoint that hands out a hostname, pinning there + * would freeze it on a name that still re-expands on every connect: no address stability gained, + * and whatever the endpoint does instead of consulting its own source once pinned is lost. + */ + private static EndPoint pin(EndPoint endPoint, SocketAddress resolvedAddress) { + if (resolvedAddress instanceof InetSocketAddress + && ((InetSocketAddress) resolvedAddress).isUnresolved()) { + return endPoint; + } + return endPoint instanceof PinnableEndPoint + ? ((PinnableEndPoint) endPoint).pinTo(resolvedAddress) + : endPoint; + } + @VisibleForTesting ChannelInitializer initializer( EndPoint endPoint, @@ -463,7 +1621,11 @@ protected void initChannel(Channel channel) { context.getNettyOptions().afterChannelInitialized(channel); } catch (Throwable t) { // If the init handler throws an exception, Netty swallows it and closes the channel. We - // want to propagate it instead, so fail the outer future (the result of connect()). + // want to propagate it instead, so fail this candidate's future. Note that is the + // per-address one, not the result of connect(): a pipeline failure that is not specific to + // the address (a bad truststore, say) therefore advances to the next candidate and is + // retried against each of them, which tryNextCandidate() documents as the deliberate + // trade-off for not being able to tell the two apart. resultFuture.completeExceptionally(t); throw t; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java new file mode 100644 index 00000000000..fb09f153dc3 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ConnectHook.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import java.util.concurrent.CompletionStage; + +/** + * A caller-supplied step that runs against every candidate channel a {@link ChannelFactory#connect} + * attempt opens, after protocol initialization succeeds and before the attempt is considered + * successful. + * + *

Its position is what makes it useful: a connect attempt may try several addresses when the + * endpoint's name resolves to more than one, and the hook runs while the factory still holds the + * remaining ones. Completing the returned stage exceptionally (or throwing synchronously) rejects + * the candidate -- the factory closes the channel and moves on to the endpoint's next address -- so + * a caller can impose its own acceptance criteria on a channel, per address, without losing the + * fallback. The control connection uses this to read {@code system.local} and refuse a channel + * whose node cannot identify itself, channeling what it read straight into its own state (see + * {@code ControlConnection}). + * + *

Contract: + * + *

+ * + *

When the options also request protocol events ({@link DriverChannelOptions#eventTypes}), the + * {@code REGISTER} request is sent after the hook completes successfully, so a channel that is + * about to be rejected never registers for events. + */ +public interface ConnectHook { + + /** + * Vets a candidate channel that completed protocol initialization. + * + * @return a stage that completes normally to accept the channel, or exceptionally to reject it + * and make the connect attempt move on to the endpoint's next address. + */ + CompletionStage onConnect(DriverChannel channel); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java index 378fd2dc0b8..7f8d768153d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/DriverChannelOptions.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; +import java.time.Duration; import java.util.Collections; import java.util.List; import net.jcip.annotations.Immutable; @@ -54,17 +55,41 @@ public static Builder builder() { */ public final boolean reportConfig; + /** + * A step the caller runs against every candidate channel after protocol initialization, with the + * power to reject it while {@link ChannelFactory} still holds the endpoint's other addresses, or + * {@code null} if the caller has no vetting to do. Precedent for a behavioral member here: {@link + * #eventCallback}. + * + *

The control connection supplies one when connecting to a node whose {@code host_id} is not + * yet known -- a contact point, the one case with something to learn -- to read {@code + * system.local} and refuse a channel whose node cannot identify itself. + * + * @see ConnectHook + */ + public final ConnectHook connectHook; + + /** + * How long the factory waits for {@link #connectHook}'s stage before treating the candidate as + * rejected. Never null when {@link #connectHook} is set. + */ + public final Duration connectHookTimeout; + private DriverChannelOptions( CqlIdentifier keyspace, List eventTypes, EventCallback eventCallback, String ownerLogPrefix, - boolean reportConfig) { + boolean reportConfig, + ConnectHook connectHook, + Duration connectHookTimeout) { this.keyspace = keyspace; this.eventTypes = eventTypes; this.eventCallback = eventCallback; this.ownerLogPrefix = ownerLogPrefix; this.reportConfig = reportConfig; + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; } public static class Builder { @@ -73,6 +98,8 @@ public static class Builder { private EventCallback eventCallback = null; private String ownerLogPrefix = null; private boolean reportConfig = false; + private ConnectHook connectHook = null; + private Duration connectHookTimeout = null; public Builder withKeyspace(CqlIdentifier keyspace) { this.keyspace = keyspace; @@ -100,9 +127,27 @@ public Builder reportConfig(boolean reportConfig) { return this; } + /** + * Arms a step that vets every candidate channel after protocol initialization, bounded by the + * given timeout. See {@link ConnectHook} for the contract. + */ + public Builder withConnectHook(ConnectHook connectHook, Duration connectHookTimeout) { + Preconditions.checkNotNull(connectHook); + Preconditions.checkNotNull(connectHookTimeout); + this.connectHook = connectHook; + this.connectHookTimeout = connectHookTimeout; + return this; + } + public DriverChannelOptions build() { return new DriverChannelOptions( - keyspace, eventTypes, eventCallback, ownerLogPrefix, reportConfig); + keyspace, + eventTypes, + eventCallback, + ownerLogPrefix, + reportConfig, + connectHook, + connectHookTimeout); } } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java index dd7630a6530..042006ea9a6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java @@ -52,7 +52,6 @@ import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,7 +158,6 @@ private enum Step { GET_CLUSTER_NAME, SET_KEYSPACE, AUTH_RESPONSE, - REGISTER, } private class InitRequest extends ChannelHandlerRequest { @@ -170,12 +168,10 @@ private class InitRequest extends ChannelHandlerRequest { private Message request; private Authenticator authenticator; private ByteBuffer authResponseToken; - private final List registerEventTypes; InitRequest(ChannelHandlerContext ctx) { super(ctx, timeoutMillis); this.step = querySupportedOptions ? Step.OPTIONS : Step.STARTUP; - this.registerEventTypes = options.eventTypes; } @Override @@ -206,8 +202,6 @@ Message getRequest() { return request = new Query("USE " + options.keyspace.asCql(false)); case AUTH_RESPONSE: return request = new AuthResponse(authResponseToken); - case REGISTER: - return request = new Register(registerEventTypes); default: throw new AssertionError("unhandled step: " + step); } @@ -330,21 +324,11 @@ void onResponse(Message response) { if (options.keyspace != null) { step = Step.SET_KEYSPACE; send(); - } else if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); } else { setConnectSuccess(); } } } else if (step == Step.SET_KEYSPACE && response instanceof SetKeyspace) { - if (!registerEventTypes.isEmpty()) { - step = Step.REGISTER; - send(); - } else { - setConnectSuccess(); - } - } else if (step == Step.REGISTER && response instanceof Ready) { setConnectSuccess(); } else if (response instanceof Error) { Error error = (Error) response; @@ -366,17 +350,6 @@ void onResponse(Message response) { } else if (step == Step.SET_KEYSPACE && error.code == ProtocolConstants.ErrorCode.INVALID) { fail(new InvalidKeyspaceException(error.message)); - } else if (step == Step.REGISTER - && error.code == ErrorCode.PROTOCOL_ERROR - && error.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { - // The server rejected CLIENT_ROUTES_CHANGE as an unknown event type. - // Fail the connection so that the caller (ClientRoutesTopologyMonitor.init()) - // gets a clear error instead of silently degrading. - fail( - "Server does not support CLIENT_ROUTES_CHANGE event " - + "(requires ScyllaDB Enterprise >= 2026.1). " - + "Either upgrade the server or remove the client routes configuration.", - null); } else { failOnUnexpected(error); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java index 5b4ff4dcec8..b319186910d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java @@ -66,7 +66,21 @@ public interface NettyOptions { /** * A hook invoked each time the driver creates a client bootstrap in order to open a channel. This - * is a good place to configure any custom option on the bootstrap. + * is a good place to configure any custom option, attribute, or {@link + * Bootstrap#resolver(io.netty.resolver.AddressResolverGroup)} on the bootstrap. + * + *

The hook runs once per logical connection to a node. When a hostname expands to several IP + * addresses, the same bootstrap is shared by every per-address attempt (each attempt uses a + * {@link Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it); likewise, protocol-version + * downgrade retries reuse it. Before multi-address support the hook ran once per attempt, + * including once per downgrade retry. + * + *

The bootstrap does not carry the driver's channel handler yet, and a handler + * installed by this hook is not honoured: the driver sets its own handler on each + * per-attempt copy afterwards (and logs a one-time warning if it overwrites one). To customize + * the pipeline, use {@link #afterChannelInitialized(Channel)} instead. (Before multi-address + * support the hook ran after the driver's handler was installed, so replacing it was technically + * possible; that was never a supported extension point.) */ void afterBootstrapInitialized(Bootstrap bootstrap); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 9da3a2a8aa2..1081dfd2538 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -34,17 +34,21 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.ClientRoutesUpdateEvent; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.DefaultTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; +import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metadata.TopologyEvent; import com.datastax.oss.driver.internal.core.util.Loggers; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.internal.core.util.concurrent.Reconnection; import com.datastax.oss.driver.internal.core.util.concurrent.RunOrSchedule; import com.datastax.oss.driver.internal.core.util.concurrent.UncaughtExceptions; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.Message; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -55,6 +59,7 @@ import com.datastax.oss.protocol.internal.response.event.TopologyChangeEvent; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; +import java.time.Duration; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collection; @@ -276,7 +281,9 @@ private class SingleThreaded { private boolean closeWasCalled; private final ReconnectionPolicy reconnectionPolicy; private final Reconnection reconnection; - private DriverChannelOptions channelOptions; + // Computed once in init() and kept; the options themselves are built fresh for every connect + // attempt (see buildChannelOptions), so this is the only part of them that lives here. + private ImmutableList eventTypes; private volatile ControlNodeState controlNodeState = ControlNodeState.NONE; // The last events received for each node private final Map lastNodeDistance = new WeakHashMap<>(); @@ -320,15 +327,8 @@ private void init( try { boolean listenClientRoutesEvents = context.getTopologyMonitor() instanceof ClientRoutesTopologyMonitor; - ImmutableList eventTypes = - buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); + this.eventTypes = buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); LOG.debug("[{}] Initializing with event types {}", logPrefix, eventTypes); - channelOptions = - DriverChannelOptions.builder() - .withEvents(eventTypes, ControlConnection.this) - .withOwnerLogPrefix(logPrefix + "|control") - .reportConfig(true) - .build(); Queue nodes = context.getLoadBalancingPolicyWrapper().newControlReconnectionQueryPlan(); @@ -393,19 +393,25 @@ private void connect( onFailure.accept(AllNodesFailedException.fromErrors(errors)); } else { LOG.debug("[{}] Trying to establish a connection to {}", logPrefix, node); + NodeInfoHolder capturedNodeInfo = new NodeInfoHolder(); context .getChannelFactory() - .connect(node, channelOptions) + .connect(node, buildChannelOptions(node, capturedNodeInfo)) .whenCompleteAsync( (channel, error) -> { try { - NodeDistance lastDistance = lastNodeDistance.get(node); - NodeState lastState = lastNodeState.get(node); + String exclusion = exclusionReason(node); if (error != null) { if (closeWasCalled || initFuture.isCancelled()) { onSuccess.run(); // abort, we don't really care about the result } else { - if (error instanceof AuthenticationException) { + // isAuthOnly, not a bare instanceof: ChannelFactory reports one failure + // per contact point with the other addresses' failures attached as + // suppressed, and it deliberately surfaces an authentication failure over + // transport ones. A name whose records failed [refused, refused, auth] is + // not an authentication problem, and logging it as one would hide that two + // thirds of the deployment is unreachable. + if (isAuthOnly(error)) { Loggers.warnWithException( LOG, "[{}] Authentication error", logPrefix, error); } else { @@ -439,22 +445,12 @@ private void connect( channel); channel.forceClose(); onSuccess.run(); - } else if (lastDistance == NodeDistance.IGNORED) { + } else if (exclusion != null) { LOG.debug( - "[{}] New channel opened ({}) but node became ignored, " - + "closing and trying next node", + "[{}] New channel opened ({}) but {}, closing and trying next node", logPrefix, - channel); - channel.forceClose(); - connect(nodes, errors, onSuccess, onFailure); - } else if (lastNodeState.containsKey(node) - && (lastState == null /*(removed)*/ - || lastState == NodeState.FORCED_DOWN)) { - LOG.debug( - "[{}] New channel opened ({}) but node was removed or forced down, " - + "closing and trying next node", - logPrefix, - channel); + channel, + exclusion); channel.forceClose(); connect(nodes, errors, onSuccess, onFailure); } else { @@ -469,7 +465,7 @@ private void connect( previousChannel); previousChannel.forceClose(); } - resolveChannelNodeIfNeeded(channel, (DefaultNode) node) + resolveChannelNodeIfNeeded(channel, (DefaultNode) node, capturedNodeInfo) .whenCompleteAsync( (resolvedNode, fetchError) -> { if (fetchError != null) { @@ -500,19 +496,58 @@ private void connect( new Exception("Channel closed during endpoint resolve"))); connect(nodes, newErrors, onSuccess, onFailure); } else { - controlNodeState = new ControlNodeState(resolvedNode, null); - context - .getEventBus() - .fire(ChannelEvent.channelOpened(resolvedNode)); - channel - .closeFuture() - .addListener( - f -> - adminExecutor - .submit( - () -> onChannelClosed(channel, resolvedNode)) - .addListener(UncaughtExceptions::log)); - onSuccess.run(); + // The guards above ran against the node the query plan offered. + // For a contact point appended by the reconnection fallback that + // is an ephemeral instance which is never the subject of a + // distance or state event, so it is never a key in either map and + // those guards cannot have seen anything. Only now, once the + // handshake has said which node actually answered, is there + // something to ask about -- and asking matters, because nothing + // downstream will: an unchanged distance fires no event, so a + // control connection parked on an excluded node stays there. + String resolvedExclusion = exclusionReason(resolvedNode); + if (resolvedExclusion != null) { + LOG.debug( + "[{}] Channel {} turned out to be {}, which {}; " + + "closing and trying next node", + logPrefix, + channel, + resolvedNode, + resolvedExclusion); + controlNodeState = ControlNodeState.NONE; + // Null out before forceClose(), as above, so that + // onChannelClosed() does not start a redundant reconnection on + // top of the connect() retry below. + ControlConnection.this.channel = null; + channel.forceClose(); + // Recorded, so that exhausting the plan this way reports why + // rather than a bare NoNodeAvailableException. No + // controlConnectionFailed event though: nothing failed to + // connect, the node is simply not one we may use -- same as the + // pre-handshake exclusion branch above. + List> newErrors = + (errors == null) ? new ArrayList<>() : errors; + newErrors.add( + new SimpleEntry<>( + resolvedNode, + new IllegalStateException(resolvedExclusion))); + connect(nodes, newErrors, onSuccess, onFailure); + } else { + controlNodeState = new ControlNodeState(resolvedNode, null); + context + .getEventBus() + .fire(ChannelEvent.channelOpened(resolvedNode)); + channel + .closeFuture() + .addListener( + f -> + adminExecutor + .submit( + () -> + onChannelClosed(channel, resolvedNode)) + .addListener(UncaughtExceptions::log)); + onSuccess.run(); + } } }, adminExecutor); @@ -530,28 +565,123 @@ private void connect( } /** - * Resolves the identity of the node at the other end of the channel. For contact point nodes - * (no hostId), queries system.local and registers a new metadata node. For nodes that already - * have a hostId, returns the node as-is. + * Why {@code candidate} must not be used for the control connection -- the load balancing + * policy has excluded it, or a topology event has -- or {@code null} if nothing rules it out. + * + *

Both maps are keyed on the {@link Node} instance and filled only from events, so a node + * that has never been the subject of one is simply absent, and absent means "nothing known + * against it" rather than "fine". That distinction is why this is asked twice per connect: once + * about the node the query plan offered, and again about the node the handshake proved is at + * the other end, which for a contact point is a different instance. */ - private CompletionStage resolveChannelNodeIfNeeded( - DriverChannel channel, DefaultNode node) { - if (node.getHostId() != null) { - return CompletableFuture.completedFuture(node); + private String exclusionReason(Node candidate) { + if (lastNodeDistance.get(candidate) == NodeDistance.IGNORED) { + return "node became ignored"; } + if (lastNodeState.containsKey(candidate)) { + NodeState state = lastNodeState.get(candidate); + if (state == null /*(removed)*/ || state == NodeState.FORCED_DOWN) { + return "node was removed or forced down"; + } + } + return null; + } + + /** + * Options for one connect attempt against one query-plan node. Built fresh per attempt rather + * than cached: an attempt against an unidentified node carries a stateful holder that the + * connect hook fills, and overlapping connect chains are reachable -- the initial {@code + * connect()} runs outside {@code Reconnection} (which serializes only its own attempts), and + * {@code reconnectNow()} checks only {@code initWasCalled} -- so nothing stateful may be shared + * between attempts. + */ + private DriverChannelOptions buildChannelOptions(Node node, NodeInfoHolder capturedNodeInfo) { + DriverChannelOptions.Builder builder = + DriverChannelOptions.builder() + .withEvents(eventTypes, ControlConnection.this) + .withOwnerLogPrefix(logPrefix + "|control") + .reportConfig(true); + if (node.getHostId() == null) { + // A contact point: the driver does not yet know which node answers at each of its + // addresses, so the identity read happens through the connect hook, inside the factory's + // candidate loop, where an address that cannot identify itself is rejected while the + // hostname's other addresses are still on hand. Read after the connect instead, the + // candidates are already gone and a failure writes off the whole plan entry. + Duration hookTimeout = + config.getDefaultProfile().getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT); + builder.withConnectHook( + channel -> readChannelNodeInfo(channel, capturedNodeInfo), hookTimeout); + } + return builder.build(); + } + + /** + * The connect hook of a contact-point attempt: reads which node answered and channels it + * straight into the attempt's holder, rejecting the candidate when the node cannot identify + * itself. + * + *

Runs on the channel's event loop and touches no {@code SingleThreaded} state: the holder + * is the only thing written, and it is read back on the admin thread only after the connect + * completes. + */ + private CompletionStage readChannelNodeInfo( + DriverChannel channel, NodeInfoHolder capturedNodeInfo) { return context .getTopologyMonitor() .getChannelNodeInfo(channel) - .thenComposeAsync( + .thenAccept( nodeInfo -> { - EndPoint resolvedEp = nodeInfo.getEndPoint(); - if (resolvedEp != null && !resolvedEp.equals(channel.getEndPoint())) { - channel.setEndPoint(resolvedEp); - LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, resolvedEp); - } - return context.getMetadataManager().registerNode(nodeInfo); - }, - adminExecutor); + // Mirrors DefaultTopologyMonitor's own precondition, so that a custom monitor + // cannot smuggle a null past registerNode: rejecting here costs one address, while + // failing in registerNode later would cost the whole plan entry. + Objects.requireNonNull( + nodeInfo.getHostId(), + "Node info is missing its host id; the node may still be bootstrapping"); + capturedNodeInfo.set(channel, nodeInfo); + }); + } + + /** + * Resolves the identity of the node at the other end of the channel. For nodes that already + * have a hostId, returns the node as-is. For a contact point, the connect hook has already read + * {@code system.local} and captured the result (see {@link #readChannelNodeInfo}); this + * registers a new metadata node from it. + */ + private CompletionStage resolveChannelNodeIfNeeded( + DriverChannel channel, DefaultNode node, NodeInfoHolder capturedNodeInfo) { + if (node.getHostId() != null) { + return CompletableFuture.completedFuture(node); + } + NodeInfo captured = capturedNodeInfo.getFor(channel); + // The pairing with the channel is asserted rather than assumed, and a miss falls back to a + // direct read: a ChannelFactory subclass that does not run the connect hook still gets a + // functioning control connection (and the mocked factories in the unit tests exercise this + // same path). The fallback costs one extra round trip, on that path only. + CompletionStage nodeInfoFuture = + (captured != null) + ? CompletableFuture.completedFuture(captured) + : context.getTopologyMonitor().getChannelNodeInfo(channel); + return nodeInfoFuture.thenComposeAsync( + nodeInfo -> { + EndPoint resolvedEp = nodeInfo.getEndPoint(); + EndPoint channelEp = channel.getEndPoint(); + // The channel adopts the node's endpoint so that everything reading it afterwards -- + // DefaultTopologyMonitor's localEndPoint on the next refresh, refreshNode's + // control-node + // check, OptionalLocalDcHelper's endpoint fallback -- sees the same instance the node + // holds, rather than the contact point this connection happened to come up through. + // + // The same test as DefaultNode#setEndPoint, and deliberately not equals(): this is + // exactly the mixed unresolved-vs-resolved case (see PinnableEndPoint#sameIdentity). + if (resolvedEp != null + && resolvedEp != channelEp + && !PinnableEndPoint.sameIdentity(resolvedEp, channelEp)) { + channel.setEndPoint(resolvedEp); + LOG.debug("[{}] Control channel endpoint upgraded to {}", logPrefix, resolvedEp); + } + return context.getMetadataManager().registerNode(nodeInfo); + }, + adminExecutor); } private void onSuccessfulReconnect() { @@ -687,10 +817,44 @@ private boolean isControlNode(Node eventNode) { && eventNode.getHostId().equals(state.current.getHostId())) { return true; } - if (state.current == null - && state.pending != null - && Objects.equals(eventNode.getEndPoint(), state.pending.getEndPoint())) { - return true; + if (state.current == null && state.pending != null) { + // Resolution is still in flight, so there is no host id to compare yet and the endpoint is + // all there is to go on. The channel's own endpoint is what to compare against: unlike the + // pending node's, ChannelFactory has bound it to the one address the connection actually + // went to. + DriverChannel pendingChannel = ControlConnection.this.channel; + if (pendingChannel == null) { + return false; + } + EndPoint eventEndPoint = eventNode.getEndPoint(); + EndPoint channelEndPoint = pendingChannel.getEndPoint(); + // Two lookup-free comparisons, because neither shape is covered by the other. + // + // resolve() settles it when both sides hold a concrete address: the event carries a + // metadata + // node whose endpoint is a resolved IP, and the channel's is pinned to the IP it reached. + // This is the case the plain hostname contact point hits, and comparing resolve() results + // rather than the endpoints keeps DefaultEndPoint#equals -- which resolves the unresolved + // side of a mixed comparison, i.e. a blocking DNS lookup on the admin thread, on an + // arbitrary single address (issue #1006) -- off a path that runs for every distance or + // state + // event arriving during a control connect. + if (Objects.equals(eventEndPoint.resolve(), channelEndPoint.resolve())) { + return true; + } + // But resolve() cannot settle it when the endpoint's current address is a *name*, which is + // the permanent state of an SNI proxy address, of a client route, and of anything a custom + // AddressTranslator hands over unresolved: the event node resolves to that unresolved name + // while the channel resolves to the IP it was pinned to, and an InetSocketAddress carrying + // an InetAddress never equals one that does not. For those endpoints identity does not key + // on the address at all -- SniEndPoint on proxy + serverName, ClientRoutesEndPoint on the + // host id, both of which a Cloud contact point and its metadata node share -- so equals() + // answers exactly the right question, and does so without resolving anything. Every + // implementation except DefaultEndPoint qualifies; that one is excluded precisely because + // its equals() is the one that would resolve. + return eventEndPoint.getClass() == channelEndPoint.getClass() + && !(eventEndPoint instanceof DefaultEndPoint) + && eventEndPoint.equals(channelEndPoint); } return false; } @@ -752,24 +916,97 @@ private void forceClose() { } } - private boolean isAuthFailure(Throwable error) { - if (error instanceof AllNodesFailedException) { - Collection> errors = - ((AllNodesFailedException) error).getAllErrors().values(); - if (errors.isEmpty()) { - return false; - } - for (List nodeErrors : errors) { - for (Throwable nodeError : nodeErrors) { - if (!(nodeError instanceof AuthenticationException)) { - return false; - } + /** + * Whether every contact point failed for the one reason worth telling the operator to go and fix + * their configuration over: bad credentials, everywhere. + * + *

Each entry is tested with {@link #isAuthOnly} rather than a bare {@code instanceof}, because + * one entry no longer means one address. {@code ChannelFactory} expands a contact-point hostname + * to every address it resolves to and reports a single failure for the name, with the other + * addresses' failures attached as suppressed exceptions. Looking only at the top-level throwable + * would call a name whose records failed {@code [refused, refused, auth]} an authentication + * failure, and claim in the log that authentication is what is wrong with the deployment when two + * thirds of it is unreachable. + */ + @VisibleForTesting + static boolean isAuthFailure(Throwable error) { + if (!(error instanceof AllNodesFailedException)) { + // Anything else carries no per-node breakdown to inspect, so there is nothing here that says + // every contact point rejected the credentials. + return false; + } + Collection> errors = ((AllNodesFailedException) error).getAllErrors().values(); + if (errors.isEmpty()) { + return false; + } + for (List nodeErrors : errors) { + for (Throwable nodeError : nodeErrors) { + if (!isAuthOnly(nodeError)) { + return false; } } } return true; } + /** Whether {@code error} and every failure attached to it are authentication failures. */ + @VisibleForTesting + static boolean isAuthOnly(Throwable error) { + if (!(error instanceof AuthenticationException)) { + return false; + } + for (Throwable suppressed : error.getSuppressed()) { + if (!(suppressed instanceof AuthenticationException)) { + return false; + } + } + return true; + } + + /** + * What one contact-point connect attempt learned about the node that answered: filled by the + * connect hook on the channel's event loop, read back on the admin thread once the connect + * completes. One instance per attempt -- overlapping attempts must not share it, which is why + * {@code DriverChannelOptions} are built per attempt. + */ + @VisibleForTesting + static final class NodeInfoHolder { + + /** + * The two values are published as one reference so that a reader can never see one candidate's + * node info paired with another's channel. Two separate volatile fields would not do: the + * factory's candidate loop can leave a stranded hook behind -- an attempt abandoned on the hook + * timeout, whose {@code system.local} response then arrives anyway -- and its late write would + * land in between the accepted candidate's two writes. The admin thread reading in that window + * would take the rejected candidate's node info as the accepted channel's, and the control + * connection would then register the wrong host id and endpoint for the node it is talking to. + * + *

With the pair atomic, that late write merely makes {@link #getFor} miss, which falls back + * to reading {@code system.local} again on the channel that is actually open. + */ + private volatile Capture capture; + + void set(DriverChannel channel, NodeInfo nodeInfo) { + this.capture = new Capture(channel, nodeInfo); + } + + /** The captured info if it came from {@code channel}: the pairing is asserted, not assumed. */ + NodeInfo getFor(DriverChannel channel) { + Capture current = this.capture; + return (current != null && current.channel == channel) ? current.nodeInfo : null; + } + + private static final class Capture { + final DriverChannel channel; + final NodeInfo nodeInfo; + + Capture(DriverChannel channel, NodeInfo nodeInfo) { + this.channel = channel; + this.nodeInfo = nodeInfo; + } + } + } + /** * Immutable snapshot of the control node state. Reads from any thread see a consistent pair of * (current, pending) via a single volatile read of the enclosing reference. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java index b93a16a6525..97aab92fff7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java @@ -26,7 +26,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -68,73 +67,34 @@ public OptionalLocalDcHelper( @Override @NonNull public Optional discoverLocalDc(@NonNull Map nodes) { - String localDcStr = context.getLocalDatacenter(profile.getName()); - Optional localDc; - if (localDcStr != null) { - LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); + String localDc = context.getLocalDatacenter(profile.getName()); + if (localDc != null) { + LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { - localDcStr = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); - LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); - } else { - localDc = Optional.empty(); - } - if (localDc.isPresent()) { - checkLocalDatacenterCompatibility( - localDc.get(), context.getMetadataManager().getContactPoints()); - // Also warn if the configured DC doesn't match any node in the cluster - if (!nodes.isEmpty()) { - boolean found = false; - for (Node node : nodes.values()) { - if (localDc.get().equals(node.getDatacenter())) { - found = true; - break; - } - } - if (!found) { - LOG.warn( - "[{}] Configured local DC '{}' does not match any node's datacenter" - + " (available DCs: {}); please verify your configuration", - logPrefix, - localDc.get(), - formatDcs(nodes.values())); - } - } + localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); + LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); } else { LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix); + return Optional.empty(); } - return localDc; - } - - /** - * Checks if the contact points are compatible with the local datacenter specified either through - * configuration, or programmatically. - * - *

The default implementation logs a warning when a contact point reports a datacenter - * different from the local one, and only for the default profile. - * - * @param localDc The local datacenter, as specified in the config, or programmatically. - * @param contactPoints The contact points provided when creating the session. - */ - protected void checkLocalDatacenterCompatibility( - @NonNull String localDc, Set contactPoints) { - if (profile.getName().equals(DriverExecutionProfile.DEFAULT_NAME)) { - Set badContactPoints = new LinkedHashSet<>(); - for (Node node : contactPoints) { - if (!Objects.equals(localDc, node.getDatacenter())) { - badContactPoints.add(node); + if (!nodes.isEmpty()) { + boolean found = false; + for (Node node : nodes.values()) { + if (localDc.equals(node.getDatacenter())) { + found = true; + break; } } - if (!badContactPoints.isEmpty()) { + if (!found) { LOG.warn( - "[{}] You specified {} as the local DC, but some contact points are from a different DC: {}; " - + "please provide the correct local DC, or check your contact points", + "[{}] Configured local DC '{}' does not match any node's datacenter" + + " (available DCs: {}); please verify your configuration", logPrefix, localDc, - formatNodesAndDcs(badContactPoints)); + formatDcs(nodes.values())); } } + return Optional.of(localDc); } /** diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 15d825b2efc..134b813bc48 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -20,19 +20,26 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; import java.util.UUID; -public class ClientRoutesEndPoint implements EndPoint { +public class ClientRoutesEndPoint implements PinnableEndPoint { private final UUID hostId; private final ClientRoutesTopologyMonitor topologyMonitor; private final String metricPrefix; @NonNull private final EndPoint fallbackEndPoint; + /** Kept only so that {@link #pinTo(SocketAddress)} can rebuild an identical copy. */ + @Nullable private final InetAddress broadcastInetAddress; + + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}, + * which key off the host id alone. + */ + @Nullable private final InetSocketAddress pinnedAddress; /** * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. @@ -49,12 +56,23 @@ public ClientRoutesEndPoint( @NonNull UUID hostId, @Nullable InetAddress broadcastInetAddress, @NonNull EndPoint fallbackEndPoint) { + this(topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, null); + } + + private ClientRoutesEndPoint( + @NonNull ClientRoutesTopologyMonitor topologyMonitor, + @NonNull UUID hostId, + @Nullable InetAddress broadcastInetAddress, + @NonNull EndPoint fallbackEndPoint, + @Nullable InetSocketAddress pinnedAddress) { this.topologyMonitor = Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); this.hostId = Objects.requireNonNull(hostId, "HOST uuid cannot be null"); this.fallbackEndPoint = Objects.requireNonNull(fallbackEndPoint, "Fallback endpoint cannot be null"); this.metricPrefix = buildMetricPrefix(broadcastInetAddress, hostId); + this.broadcastInetAddress = broadcastInetAddress; + this.pinnedAddress = pinnedAddress; } @NonNull @@ -62,18 +80,75 @@ public UUID getHostId() { return hostId; } + /** + * Returns the address connections should be opened to. + * + *

The client route for this host id is an in-memory lookup over the cached {@code + * system.client_routes} contents, and it yields exactly one address by design, so this neither + * blocks nor expands to several candidates. The route's hostname is returned {@linkplain + * InetSocketAddress#isUnresolved() unresolved}: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup}, so a custom resolver is honoured and no DNS lookup + * runs on the caller (the admin event loop, for control-connection reconnects). + * + *

When the topology monitor has no route for this host id — i.e. the node is not reached + * through a cloud private endpoint — this delegates to the fallback endpoint. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly. + */ @NonNull @Override public SocketAddress resolve() { + if (pinnedAddress != null) { + return pinnedAddress; + } + InetSocketAddress address; try { - InetSocketAddress address = topologyMonitor.resolve(hostId); - if (address != null) { - return address; - } - } catch (IOException e) { - throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); + address = topologyMonitor.resolve(hostId); + } catch (IllegalStateException e) { + // The monitor is closed, so its route cache is gone -- but resolve() still has to answer, and + // the honest answer is "no route available", which is what the fallback endpoint is for. + // Throwing here is not contained anywhere useful: PinnableEndPoint#sameIdentity compares + // resolve() results for every node of every topology refresh, and neither NodesRefresh nor + // MetadataManager#apply catches, so a refresh that raced session shutdown would be dropped + // whole and surface only as a DEBUG log in ControlConnection#onSuccessfulReconnect. + address = null; } - return fallbackEndPoint.resolve(); + return address != null ? address : fallbackEndPoint.resolve(); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirror DefaultEndPoint: an address we cannot hold in an InetSocketAddress field skips + // pinning rather than failing the connection. So does an unresolved one -- resolve() hands out + // the route's hostname unresolved, and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze the endpoint on + // a name that still re-expands on every connect: no address stability gained, and the route + // lookup silenced for good, since resolve() short-circuits once pinned. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; + } + return new ClientRoutesEndPoint( + topologyMonitor, + hostId, + broadcastInetAddress, + fallbackEndPoint, + (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true}: a route's addresses are alternative ways in to this one node, so connections + * may be spread across them. + */ + @Override + public boolean addressesAreInterchangeable() { + return true; } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index 1ffc35fd9f4..a189f16c565 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -21,6 +21,7 @@ import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; @@ -32,9 +33,9 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; @@ -195,9 +196,18 @@ void setResolvedRoutes(Map routes) { resolvedRoutesCache.set(Collections.unmodifiableMap(new HashMap<>(routes))); } + /** + * Returns the client route for {@code hostId} as an {@linkplain InetSocketAddress#isUnresolved() + * unresolved} address, or {@code null} if this node has no route. + * + *

The route's hostname is deliberately left unresolved: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup} at connection time. That keeps this method a pure + * in-memory cache lookup, so it is safe to call from an event loop, and it means a custom + * resolver applies to client routes just like it does to contact points. + */ @Nullable - public InetSocketAddress resolve(@NonNull UUID hostId) - throws IllegalStateException, UnknownHostException { + public InetSocketAddress resolve(@NonNull UUID hostId) throws IllegalStateException { if (closed) { throw new IllegalStateException("Topology monitor is closed"); } @@ -206,7 +216,7 @@ public InetSocketAddress resolve(@NonNull UUID hostId) return null; // no client route for this node — caller falls back to default } - return new InetSocketAddress(resolveAddress(route.getHostname()), route.getPort()); + return InetSocketAddress.createUnresolved(route.getHostname(), route.getPort()); } /** @@ -480,6 +490,38 @@ protected EndPoint buildNodeEndPoint( return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress, fallback); } + @Override + public boolean reresolvesNodeAddresses() { + // ClientRoutesEndPoint hands the route hostname over unresolved, so the connection layer + // re-expands it on every connection attempt -- but only when a route exists for that host_id + // (see ClientRoutesEndPoint#resolve()); for mixed/incomplete route sets it delegates to a + // static, already-resolved fallback endpoint instead. Only report true when every + // currently-known node actually has a live route; otherwise the contact-point reconnection + // fallback must stay available for the nodes stuck on that fallback. + // + // "Every known node" has to mean at least one: with an empty node set the loop below would + // report true vacuously, suppressing the contact-point fallback at the one moment it is the + // only + // way back -- before the first node refresh, or after the monitor has removed everything. (The + // caller happens to exempt an empty query plan as well, but that is a separate safety net and + // this must not depend on it.) + // + // The answer legitimately changes as route coverage does, so successive reconnection rounds can + // see different values: that tracks reality rather than flapping. The scan is O(nodes) and runs + // once per reconnection attempt, against an in-memory map. + Collection nodes = context.getMetadataManager().getMetadata().getNodes().values(); + if (nodes.isEmpty()) { + return false; + } + Map routes = resolvedRoutesCache.get(); + for (Node node : nodes) { + if (!routes.containsKey(node.getHostId())) { + return false; + } + } + return true; + } + /** * Builds the CQL query to fetch client routes. * @@ -645,13 +687,4 @@ public CompletionStage closeAsync() { LOG.debug("[{}] ClientRoutesTopologyMonitor closed", logPrefix); return super.closeAsync(); } - - /** - * Resolves a hostname to an {@link InetAddress}. Extracted as a protected method so that unit - * tests can override it to return stubbed addresses without hitting the network. - */ - @NonNull - protected InetAddress resolveAddress(@NonNull String hostname) throws UnknownHostException { - return InetAddress.getByName(hostname); - } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java index 021824a9b16..7bdf1c4e1ec 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java @@ -44,4 +44,14 @@ protected EndPoint buildNodeEndPoint( UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); return new SniEndPoint(cloudProxyAddress, hostId.toString()); } + + @Override + public boolean reresolvesNodeAddresses() { + // Every node is reached through the cloud SNI proxy, and SniEndPoint hands the proxy hostname + // over unresolved, so the connection layer re-expands it on every connection attempt (see + // ChannelFactory#resolveCandidates). Addresses therefore stay current on their own: appending + // the original contact points as a DNS re-resolution fallback would add nothing, and could + // resurrect nodes this monitor has authoritatively removed. + return true; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 7ffbee8e4bb..ffe2e3917b6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -19,28 +19,104 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Serializable; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class DefaultEndPoint implements EndPoint, Serializable { +public class DefaultEndPoint implements PinnableEndPoint, Serializable { private static final long serialVersionUID = 1; + private static final Logger LOG = LoggerFactory.getLogger(DefaultEndPoint.class); + + /** Static, so the warning below is emitted once per JVM rather than once per endpoint. */ + private static final AtomicBoolean LOGGED_MIXED_COMPARISON_WARNING = new AtomicBoolean(); + private final InetSocketAddress address; private final String metricPrefix; + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals}, {@link #hashCode} and + * {@link #asMetricPrefix()}: a pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + public DefaultEndPoint(InetSocketAddress address) { + this(address, null); + } + + private DefaultEndPoint(InetSocketAddress address, @Nullable InetSocketAddress pinnedAddress) { this.address = Objects.requireNonNull(address, "address can't be null"); this.metricPrefix = buildMetricPrefix(address); + this.pinnedAddress = pinnedAddress; } + /** + * Returns the address connections should be opened to: the {@linkplain #pinTo(SocketAddress) + * pinned} one if this is a pinned copy, otherwise the stored address as-is. + * + *

This performs no name resolution. If the stored address is a hostname (i.e. {@linkplain + * InetSocketAddress#isUnresolved() unresolved} — contact points are always kept unresolved, see + * {@link com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}) it is returned + * unresolved, and {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it + * to every IP it maps to through Netty's configured {@code AddressResolverGroup}. Resolving there + * rather than here is deliberate: it keeps any custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized} in the + * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass, and it + * keeps this method non-blocking so it is safe to call from an event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - return address; + return pinnedAddress != null ? pinnedAddress : address; } + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); + if (!(resolvedAddress instanceof InetSocketAddress) + // An unresolved address, as ClientRoutesEndPoint and SniEndPoint also refuse: this endpoint + // hands a hostname over unresolved and ChannelFactory passes it straight back when the user + // disabled the resolver or a custom one declines it. Pinning that would freeze resolve() on + // a name that must re-expand on every connect. + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress) + // The address we already hold: pinning to it changes nothing, since resolve() and + // toString() would keep yielding what they already do. Skipping the copy spares toString() + // a + // redundant "addr(addr)" suffix on every already-resolved endpoint -- which is all of them, + // once a node is discovered from the peers rows. + || resolvedAddress.equals(this.address)) { + return this; + } + return new DefaultEndPoint(address, (InetSocketAddress) resolvedAddress); + } + + /** + * Whether {@code other} denotes the same node: the stored addresses are compared, ignoring which + * one either endpoint may be {@linkplain #pinTo(SocketAddress) pinned} to. + * + *

Comparing an unresolved address against a resolved one costs a DNS lookup, taken + * inline on the calling thread, because the unresolved side has to be resolved first. It is also + * arbitrary: {@code new InetSocketAddress(name, port)} keeps only the first address the + * name maps to, so for a multi-record name the answer is "equal iff this node is the one the + * resolver happened to list first". And it does not agree with {@link #hashCode()}, which keys on + * the stored address alone -- so a hostname and one of its IPs can be {@code equals} while + * hashing differently, and a hash-based collection of endpoints (the contact-point {@code Set}, + * for one) never treats them as the same entry. + * + *

No driver-internal path compares the two forms; the branch is kept for {@link + * com.datastax.oss.driver.api.core.metadata.Metadata#findNode(EndPoint)}, whose caller may hold + * either. It warns once per JVM, as a canary for whether anything still depends on it -- see + * https://github.com/scylladb/java-driver/issues/1006. + */ @Override public boolean equals(Object other) { if (other == this) { @@ -48,12 +124,15 @@ public boolean equals(Object other) { } else if (other instanceof DefaultEndPoint) { InetSocketAddress thisAddress = this.address; InetSocketAddress thatAddress = ((DefaultEndPoint) other).address; - // If only one of the addresses is unresolved, resolve the other. Otherwise (both resolved or - // both unresolved), compare as-is. - if (thisAddress.isUnresolved() && !thatAddress.isUnresolved()) { - thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); - } else if (thatAddress.isUnresolved() && !thisAddress.isUnresolved()) { - thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + // If only one of the addresses is unresolved, resolve it. Otherwise (both resolved or both + // unresolved), compare as-is. + if (thisAddress.isUnresolved() != thatAddress.isUnresolved()) { + warnAboutMixedComparison(thisAddress, thatAddress); + if (thisAddress.isUnresolved()) { + thisAddress = new InetSocketAddress(thisAddress.getHostName(), thisAddress.getPort()); + } else { + thatAddress = new InetSocketAddress(thatAddress.getHostName(), thatAddress.getPort()); + } } return thisAddress.equals(thatAddress); } else { @@ -61,6 +140,18 @@ public boolean equals(Object other) { } } + private static void warnAboutMixedComparison(InetSocketAddress one, InetSocketAddress other) { + if (LOGGED_MIXED_COMPARISON_WARNING.compareAndSet(false, true)) { + LOG.warn( + "Compared an unresolved endpoint address against a resolved one ({} vs {}). This performs" + + " a DNS lookup on the calling thread, only compares the first address the name maps" + + " to, and does not agree with hashCode(); see" + + " https://github.com/scylladb/java-driver/issues/1006. This message is logged once.", + one, + other); + } + } + @Override public int hashCode() { return address.hashCode(); @@ -68,6 +159,9 @@ public int hashCode() { @Override public String toString() { + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which IP a given connection + // landed on is in the channel's own toString(), which Netty builds from the actual remote + // address. return address.toString(); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java index 1b09c26ce16..16da0750f8f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java @@ -102,15 +102,105 @@ public EndPoint getEndPoint() { } public void setEndPoint(@NonNull EndPoint newEndPoint, @NonNull InternalDriverContext context) { - if (!newEndPoint.equals(endPoint)) { - endPoint = newEndPoint; - // metricUpdater is transient, so it can be null on deserialized nodes. - NodeMetricUpdater previousMetricUpdater = metricUpdater; - if (previousMetricUpdater != null - && !(previousMetricUpdater instanceof NoopNodeMetricUpdater)) { - metricUpdater = context.getMetricsFactory().newNodeUpdater(this); - previousMetricUpdater.clearMetrics(); - } + // Nothing downstream can tell the two instances apart, so keep the one already held. Not merely + // an optimization: the instance this node holds may carry a reverse-DNS name cached on its + // InetAddress by an earlier TLS handshake (DefaultSslEngineFactory calls getHostName() under + // the + // default advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true), and every full + // topology refresh mints a brand-new endpoint over a brand-new InetSocketAddress for every + // node. + // Adopting each one unconditionally would throw that name away and make the next connection to + // the node repeat the blocking reverse lookup on a Netty I/O loop -- once per node per refresh + // instead of once per node per session, during exactly the refresh-then-reconnect storms this + // feature exists for. + // + // Deliberately not equals(): see PinnableEndPoint#sameIdentity, which is also what + // ControlConnection uses when it decides whether the control channel should adopt a node's + // endpoint. + if (PinnableEndPoint.sameIdentity(newEndPoint, endPoint)) { + return; + } + + // Metrics are registered under names derived from the endpoint, so they have to be + // re-registered + // whenever those names change -- which is not the same question as whether this is a different + // node. It is narrower in one direction: a PinnableEndPoint copy differs from the original only + // by the address it is pinned to, and both equals() and the metric identity ignore that by + // contract (see PinnableEndPoint). And it is wider in the other: an unresolved hostname and the + // resolved address it maps to compare *equal* (see DefaultEndPoint#equals) while their metric + // prefixes differ, which is exactly what happens when a contact-point node adopts the endpoint + // built from its system.local row. + // + // asMetricPrefix() alone, deliberately, even though the tagging MetricIdGenerator tags metrics + // with the endpoint's toString() rather than its prefix. toString() cannot be used as an + // identity key because it is not stable across equal instances: DefaultEndPoint delegates it to + // InetSocketAddress, which renders InetAddress's *cached* hostName field, and that field is + // populated the first time anything calls getHostName(). DefaultSslEngineFactory does exactly + // that while building an engine, under the default advanced.ssl-engine-factory + // .allow-dns-reverse-lookup-san = true -- on the very instance this node holds, since an + // already-resolved endpoint is passed through unchanged by resolveCandidates() and pin(). So + // after the first channel this node's endpoint renders as "host/1.2.3.4:9042" while the one the + // next refresh decodes from system.peers renders as "/1.2.3.4:9042", and keying on that would + // clear and re-register every node's metrics on every topology refresh -- widening the + // clear/rebuild race described below from "once per endpoint change" to "always". + // + // The cost of leaving it out: a tagging generator can keep reporting under an endpoint string + // the node no longer answers to, until something else changes the prefix. + boolean differentMetricIdentity = + !newEndPoint.asMetricPrefix().equals(endPoint.asMetricPrefix()); + // metricUpdater is transient, so it can be null on deserialized nodes. + NodeMetricUpdater previousMetricUpdater = metricUpdater; + boolean rebuildMetricUpdater = + differentMetricIdentity + && previousMetricUpdater != null + && !(previousMetricUpdater instanceof NoopNodeMetricUpdater); + + // Clearing comes *before* the swap. Dropwizard and MicroProfile do not remember the ids they + // registered under; clearMetrics() recomputes each one from this node's current endpoint (see + // DropwizardMetricUpdater#clearMetrics and MetricIdGenerator#nodeMetricId). Clearing after the + // swap would therefore delete the series the new updater had just registered and leave the old + // ones behind, under a name nothing writes to any more. Micrometer removes the Meter instances + // it holds and does not care either way. + // + // The three steps are not atomic with respect to concurrent metric writes: metricUpdater is + // volatile and read from I/O threads, so a write landing between the clear and the rebuild + // goes through the updater that was just cleared, and Dropwizard re-registers on demand + // (getOrCreateCounterFor -> registry.counter(getMetricId(m))). That resurrects one series, + // named from whichever endpoint this node holds at that instant. + // + // Note the window is narrow but its effect is not transient: the resurrected metric is cached + // in the old updater's map, and ChannelFactory snapshots node.getMetricUpdater() once per + // connection and hands it to the traffic meters, which hold it for the channel's life. So a + // mark that lands here keeps reporting under the old endpoint's name until every channel open + // at that moment has been recycled. Nothing throws -- registry.counter() is get-or-create -- + // and the request path re-reads getMetricUpdater() per request, so the misreporting is + // confined to the byte counters. Closing it properly means having clearMetrics() remove the + // ids it registered under rather than recomputing them from the current endpoint, which is a + // change to every metrics implementation and would also fix a second problem: two nodes can + // briefly share a metric prefix (a control node's endpoint is its contact point's), and then + // this clear deletes the series the other node just registered. + if (rebuildMetricUpdater) { + previousMetricUpdater.clearMetrics(); + } + + // Adopt the newest instance even when it compares equal: a pinned copy carries the address + // every + // subsequent connection to this node will use, so refusing it would freeze the node on the + // first + // address it ever connected to, even after the control connection moved to another one and told + // us about it. (The early return above lets through exactly the instances that differ in that + // address, or in metric identity, or in kind.) + endPoint = newEndPoint; + + // And building comes *after* it: the updaters register every enabled metric from their + // constructor, deriving the names from the endpoint this node holds at that moment. + if (rebuildMetricUpdater) { + NodeMetricUpdater newMetricUpdater = context.getMetricsFactory().newNodeUpdater(this); + // Carry over any pending metrics expiration before publishing the replacement: the factories + // arm and cancel it through node.getMetricUpdater(), so from here on they would only ever + // reach the new one, leaving the old one's timer pending on an object nothing refers to. + newMetricUpdater.adoptExpirationFrom(previousMetricUpdater); + metricUpdater = newMetricUpdater; } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 5a82bfe2c86..f3761733c32 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -352,22 +352,25 @@ public CompletionStage getChannelNodeInfo(DriverChannel channel) { } EndPoint localEndPoint = channel.getEndPoint(); return query(channel, buildQuery(localColumns, "system.local", "key='local'")) - .thenApply( - result -> { - if (localColumns == null && !result.getColumnNames().isEmpty()) { - localColumns = - intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); - } - Iterator iterator = result.iterator(); - if (!iterator.hasNext()) { - throw new IllegalStateException( - "Expected a row in system.local for node info resolution, got empty result"); - } - AdminRow localRow = iterator.next(); - InetSocketAddress broadcastRpcAddress = - getBroadcastRpcAddress(localRow, localEndPoint); - return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); - }); + .thenApply(result -> toLocalNodeInfo(result, localEndPoint)); + } + + /** + * Decodes the single {@code system.local} row of {@code result} into a {@link NodeInfo}, warming + * the local column cache from it on the way. + */ + private NodeInfo toLocalNodeInfo(AdminResult result, EndPoint localEndPoint) { + if (localColumns == null && !result.getColumnNames().isEmpty()) { + localColumns = intersectWithNeeded(result.getColumnNames(), LOCAL_COLUMNS_OF_INTEREST); + } + Iterator iterator = result.iterator(); + if (!iterator.hasNext()) { + throw new IllegalStateException( + "Expected a row in system.local for node info resolution, got empty result"); + } + AdminRow localRow = iterator.next(); + InetSocketAddress broadcastRpcAddress = getBroadcastRpcAddress(localRow, localEndPoint); + return nodeInfoBuilder(localRow, broadcastRpcAddress, localEndPoint).build(); } @Override @@ -659,8 +662,52 @@ protected EndPoint buildNodeEndPoint( // Don't rely on system.local.rpc_address for the control node, because it mistakenly // reports the normal RPC address instead of the broadcast one (CASSANDRA-11181). We // already know the endpoint anyway since we've just used it to query. + return connectedNodeEndPoint(localEndPoint); + } + } + + /** + * The endpoint to register the connected node under: the address the control channel actually + * reached, rather than the contact point it was reached through. + * + *

They differ when the control connection came up through a contact point, because a contact + * point is kept unresolved and {@code ChannelFactory} binds a {@linkplain PinnableEndPoint + * pinned} copy of it to the one address the channel reached -- a copy that, by that interface's + * contract, is identified exactly like the unpinned original. Registering the node under it would + * give it a hostname identity: metric names and tags derived from a name that denotes the + * whole cluster rather than this node. That is bad enough on its own, but the real damage is that + * the identity is not the node's: the reconnection fallback hands the contact points back on + * every reconnection round, so each successive control node acquires the same one. Two live nodes + * then report under a single metric prefix, sharing get-or-create metric objects, until the next + * refresh moves the older one back to its own address -- and {@code clearMetrics()} recomputes + * the names to delete from the prefix the node still holds, taking the newcomer's freshly + * registered series with it (see {@code DefaultNode#setEndPoint}). + * + *

Deriving the identity from the address actually connected to fixes all of that at once: it + * is this node's own address, so it is unique to it, and re-registering an unchanged control node + * becomes a no-op instead of an identity change. + * + *

Endpoints this cannot rebuild are returned untouched -- a third-party {@link EndPoint}, or + * one whose {@code resolve()} is not a resolved {@code InetSocketAddress} (the user disabled + * Netty's resolver, or a custom one declined the address, so nothing was pinned). So is one that + * already carries the connected address, which is every reconnection to an identified node: the + * existing instance is kept so that the control node's endpoint stays {@code ==} to the + * channel's, which is what keeps {@link #refreshNode}'s control-node check an identity + * comparison. + */ + private static EndPoint connectedNodeEndPoint(EndPoint localEndPoint) { + if (!(localEndPoint instanceof DefaultEndPoint)) { + return localEndPoint; + } + SocketAddress connected = localEndPoint.resolve(); + if (!(connected instanceof InetSocketAddress) + || ((InetSocketAddress) connected).isUnresolved()) { return localEndPoint; } + DefaultEndPoint asConnected = new DefaultEndPoint((InetSocketAddress) connected); + return asConnected.asMetricPrefix().equals(localEndPoint.asMetricPrefix()) + ? localEndPoint + : asConnected; } // Called when a new node is being added; the peers table is keyed by broadcast_address, diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index f3f3e4fe346..fd11f4c902f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -27,6 +27,8 @@ import com.datastax.oss.driver.api.core.session.Request; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.util.collection.CompositeQueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.internal.core.util.concurrent.ReplayingEventFilter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; @@ -147,7 +149,9 @@ public Queue newQueryPlan( switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: - // The contact points are not stored in the metadata yet: + // The contact points are not stored in the metadata yet. Each unresolved hostname is + // expanded to all its DNS IPs at connection time by ChannelFactory, so one entry per + // contact point is enough here. List nodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(nodes); return new ConcurrentLinkedQueue<>(nodes); @@ -164,20 +168,69 @@ public Queue newQueryPlan( @NonNull public Queue newControlReconnectionQueryPlan() { + // Read the state once, before building the regular plan. State transitions are monotonic + // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value + // newQueryPlan() reads internally; that guarantees we never both build the plan from the + // contact points (pre-RUNNING branch of newQueryPlan) and append them again below. + // + // Note: this is still two separate reads of stateRef (this one, and newQueryPlan()'s own + // internal read a moment later), so a transition landing exactly between them is possible: if + // state flips BEFORE_INIT/DURING_INIT -> RUNNING in that window, newQueryPlan() takes the + // RUNNING branch (a real LBP-built plan) while the state captured here is still pre-RUNNING, + // so the contact-point fallback below is skipped for this one call even though + // regularQueryPlan didn't come from the contact-point branch. This is benign: no crash, no + // duplicate entries, and it self-corrects on the very next reconnection attempt. + // + // Monotonicity leaves the other direction open, and it is worth naming: a RUNNING -> CLOSING + // flip in that same window makes newQueryPlan() take its default branch and return an empty + // plan, which passes both the RUNNING check below and the empty-plan exemption from the + // re-resolving-monitor rule, so the plan handed back is the contact points alone. Also benign: + // ControlConnection abandons a reconnection attempt on closeWasCalled, and every node in that + // plan is one it already had. + State state = stateRef.get(); Queue regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - if (context - .getConfig() - .getDefaultProfile() - .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) { - Set originalNodes = context.getMetadataManager().getContactPoints(); - List contactNodes = new ArrayList<>(); - for (DefaultNode node : originalNodes) { - contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context)); - } + // Only append the contact points as an explicit fallback once the LBP is RUNNING: before that + // (BEFORE_INIT/DURING_INIT), newQueryPlan() above already built regularQueryPlan directly from + // the contact points, so appending them again here would just duplicate every entry. + // + // Skipped when the topology monitor re-resolves node addresses on its own (e.g. proxy-based + // monitors such as client routes or the cloud SNI proxy): those keep addresses fresh without + // this fallback, and appending raw contact points could resurrect nodes the monitor has + // authoritatively removed. The exception is an empty regular plan: with no live node to try, + // reconnection cannot recover on its own, so the contact-point fallback is kept even for those + // monitors. + if (state == State.RUNNING + && context + .getConfig() + .getDefaultProfile() + .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS) + && (!context.getTopologyMonitor().reresolvesNodeAddresses() + || regularQueryPlan.isEmpty())) { + // Append the original (unresolved) contact points so every IP their hostname resolves to is + // tried as a fallback: ChannelFactory expands each one at connection time, instead of the + // driver being stuck with whatever single IP a metadata node happens to hold. + // + // The retained instances, not fresh copies. MetadataManager holds the contact-point nodes for + // the session's lifetime and the pre-RUNNING branch of newQueryPlan() already hands out these + // very objects, so minting a copy per plan would give each reconnection round a distinct node + // firing its own controlConnectionFailed event -- one set per round, for as long as + // reconnection lasts. Shuffling a fresh list leaves the retained set itself untouched. + // + // Metrics are not a reason either way, and are worth stating because it looks as though they + // should be: DefaultNode.newContactPoint installs NoopNodeMetricUpdater, so a contact-point + // node records nothing -- no bytes, no errors.connection.init, no errors.connection.auth -- + // and a fresh copy would be no worse. Now that this fallback is on by default, that blind + // spot covers every reconnect that reaches a contact point rather than only session init, so + // an operator watching errors.connection.auth will not see a contact point failing to + // authenticate. Giving these nodes real updaters would register metrics under names for + // ephemeral objects that are deliberately absent from metadata, so it is left as is. + List contactNodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(contactNodes); - // Append contact points to the end of the regular query plan so they serve as a fallback - regularQueryPlan.addAll(contactNodes); + // Concatenate rather than mutate: the RUNNING-state regularQueryPlan is a built-in QueryPlan + // whose add()/addAll() throw UnsupportedOperationException (poll() is its only mutator). + // CompositeQueryPlan drains the regular plan first, then the contact-point fallback. + return new CompositeQueryPlan(regularQueryPlan, new SimpleQueryPlan(contactNodes.toArray())); } return regularQueryPlan; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java index cd765c818e6..d8671678306 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java @@ -188,6 +188,12 @@ public boolean wasImplicitContactPoint() { * they are never added to metadata and never exposed to user-facing APIs (events, {@link * com.datastax.oss.driver.api.core.metadata.Metadata#getNodes()}, or {@link * com.datastax.oss.driver.api.core.metadata.NodeStateListener} callbacks). + * + *

The metadata node stores {@code nodeInfo.getEndPoint()} as-is and never re-resolves it on + * its own. Re-resolving the original contact-point hostname to pick up current DNS only happens + * through the original-contact-point reconnection fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}), which re-enters + * the contact points and lets {@code ChannelFactory} expand each hostname at connection time. */ public CompletionStage registerNode(NodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId"); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java new file mode 100644 index 00000000000..4154250ce4b --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.SocketAddress; +import java.util.Objects; + +/** + * An {@link EndPoint} that can produce a copy of itself bound ("pinned") to one specific address. + * + *

An endpoint whose hostname maps to several IPs describes a set of candidate addresses, + * but a channel is always connected to exactly one of them. {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} pins the endpoint to the address it + * actually used, and hands the pinned copy to the channel. That matters for two reasons: + * + *

+ * + *

This is an internal extension point: {@code ChannelFactory} pins endpoints that implement it + * and leaves any other implementation untouched, so third-party {@link EndPoint}s keep working + * exactly as before. + * + *

Implementations must keep {@link Object#equals}, {@link Object#hashCode}, {@link + * EndPoint#asMetricPrefix()} and {@link Object#toString()} identical to the unpinned + * original: a pinned copy denotes the same node, and every one of those is part of how the node is + * identified from the outside. Metric names in particular must not change depending on which IP a + * connection happened to land on — and that includes {@code toString()}, which is what {@code + * TaggingMetricIdGenerator} tags node metrics with, and what any third-party {@code + * MetricIdGenerator} is equally free to use. Nodes do adopt pinned copies (see {@code + * DefaultNode#setEndPoint}), so an identity that varied with the pin would silently re-tag a node's + * metrics mid-session. Equality must also stay symmetric: {@code original.equals(pinned)} and + * {@code pinned.equals(original)} must agree, since endpoints are used as set and map keys. + * + *

The pinned address is therefore observable only through {@link EndPoint#resolve()}. That is no + * loss for diagnostics: the address a channel is actually connected to appears in the channel's own + * {@code toString()}, which Netty builds from its remote address, and {@code ChannelFactory} logs + * each candidate as it tries it. + */ +public interface PinnableEndPoint extends EndPoint { + + /** + * Returns a copy of this endpoint that resolves to exactly {@code resolvedAddress}. + * + *

Implementations may return {@code this} when pinning does not apply (for example when the + * address is not of a type they can hold on to), or when it would be a no-op because the endpoint + * already resolves to exactly that address. + * + * @param resolvedAddress the address a connection was successfully established to; must not be + * null and must already be resolved. + */ + @NonNull + EndPoint pinTo(@NonNull SocketAddress resolvedAddress); + + /** + * Whether the addresses this endpoint expands to are interchangeable, i.e. reaching any one of + * them is reaching the same node. + * + *

This is what decides whether {@code ChannelFactory} may spread connections across them. The + * question is a property of what the name denotes, and it splits the name-based endpoints + * in two: + * + *

+ * + *

Only consulted for a node the driver has already identified. A contact point is spread + * across its addresses regardless, since they may well be different nodes and there is no node + * identity to preserve yet. + */ + default boolean addressesAreInterchangeable() { + return false; + } + + /** + * Whether two endpoints denote the same node and are indistinguishable to everything that + * reads one: same runtime type, same {@linkplain EndPoint#asMetricPrefix() metric identity}, same + * {@linkplain EndPoint#resolve() current address}. + * + *

Deliberately not {@link Object#equals}: {@code DefaultEndPoint#equals} resolves the + * unresolved side of a mixed comparison, which would put a blocking DNS lookup on the admin + * thread for every endpoint that is still a hostname — and contact points are now kept + * unresolved, so that is reachable (see issue #1006). It is also narrower than {@code equals} in + * one direction and wider in another, which is exactly what callers need: + * + *

+ * + *

The class check keeps a node from staying on a plain fallback endpoint when a dynamic one + * ({@code ClientRoutesEndPoint}) with the same current address arrives. + * + *

{@code toString()} is not part of the test, even though {@code TaggingMetricIdGenerator} + * tags metrics with it rather than with the prefix. It is not stable across equal instances: + * {@code DefaultEndPoint} delegates it to {@code InetSocketAddress}, which renders {@code + * InetAddress}'s cached {@code hostName} field, and that field is populated the first time + * anything calls {@code getHostName()} — which {@code DefaultSslEngineFactory} does while + * building an engine, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying on it would report a + * difference for every node on every topology refresh. The cost of leaving it out: a tagging + * generator can keep reporting under an endpoint string the node no longer answers to, until + * something else changes the prefix. + */ + static boolean sameIdentity(@NonNull EndPoint first, @NonNull EndPoint second) { + return first.getClass() == second.getClass() + && first.asMetricPrefix().equals(second.asMetricPrefix()) + && Objects.equals(first.resolve(), second.resolve()); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index d1ab8eec98d..18982f3b681 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -18,61 +18,140 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.shaded.guava.common.primitives.UnsignedBytes; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetAddress; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Comparator; +import java.net.SocketAddress; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -public class SniEndPoint implements EndPoint { - private static final AtomicInteger OFFSET = new AtomicInteger(); +public class SniEndPoint implements PinnableEndPoint { private final InetSocketAddress proxyAddress; private final String serverName; /** - * @param proxyAddress the address of the proxy. If it is {@linkplain - * InetSocketAddress#isUnresolved() unresolved}, each call to {@link #resolve()} will - * re-resolve it, fetch all of its A-records, and if there are more than 1 pick one in a - * round-robin fashion. + * The proxy IP this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}: a + * pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + + /** + * @param proxyAddress the address of the proxy. Stored {@linkplain + * InetSocketAddress#isUnresolved() unresolved}, whatever form it was supplied in, so that the + * driver expands a proxy hostname to all of its A-records at connection time and tries each + * of them — see {@link #storeUnresolved}. * @param serverName the SNI server name. In the context of Cloud, this is the string * representation of the host id. */ public SniEndPoint(InetSocketAddress proxyAddress, String serverName) { - this.proxyAddress = Objects.requireNonNull(proxyAddress, "SNI address cannot be null"); + this(proxyAddress, serverName, null); + } + + private SniEndPoint( + InetSocketAddress proxyAddress, + String serverName, + @Nullable InetSocketAddress pinnedAddress) { + this.proxyAddress = + storeUnresolved(Objects.requireNonNull(proxyAddress, "SNI address cannot be null")); this.serverName = Objects.requireNonNull(serverName, "SNI Server name cannot be null"); + this.pinnedAddress = pinnedAddress; + } + + /** + * Stores the proxy address unresolved, whatever form it arrived in. + * + *

{@link #resolve()} hands the stored address to the connection layer as-is, and only an + * unresolved one gets expanded and re-expanded there. A proxy hostname supplied already resolved + * would therefore stay bound to whichever single IP its lookup happened to return, for the life + * of the session: no spreading across the proxy's A-records, no fallback when that one IP stops + * answering, and no pick-up of a DNS change. That is a real possibility for a hostname handed to + * {@link + * com.datastax.oss.driver.api.core.session.SessionBuilder#withCloudProxyAddress(InetSocketAddress)}, + * because the ordinary {@code InetSocketAddress(String, int)} constructor resolves eagerly. + * ({@code CloudConfigFactory}, the usual path, already builds an unresolved address.) + * + *

An address that is already an IP literal is stored unresolved too, even though it has + * nothing to expand, because that is what makes this endpoint's identity stable. A + * resolved address's {@code getHostString()} is not fixed: it starts out as the IP literal and + * begins reporting the reverse-DNS name as soon as anything calls {@code getHostName()} on the + * underlying {@code InetAddress} — which {@code SniSslEngineFactory#newSslEngine} does, on this + * very instance, under the default {@code + * advanced.ssl-engine-factory.allow-dns-reverse-lookup-san = true}. Keying {@link #equals} and + * {@link #asMetricPrefix()} off a string that can change underneath them would move a node's + * metrics mid-session and make endpoints built before and after the first TLS handshake compare + * unequal. An unresolved address has no such field to fill in: its host string is fixed at + * construction, and {@code getHostName()} on it performs no lookup. SNI's reverse lookup still + * happens, on the {@linkplain #pinTo(SocketAddress) pinned} copy that carries the IP the channel + * actually reached. + * + *

What this cannot defend against is an instance the caller polluted before handing it over, + * i.e. called {@code getHostName()} on themselves. + * + *

Normalizing here rather than at the call site keeps every {@code SniEndPoint} built from the + * same proxy comparable — {@link #equals} keys on this field — and matches what this endpoint did + * before resolution moved to the connection layer, when it re-resolved the proxy hostname on + * every {@code resolve()} call. + */ + private static InetSocketAddress storeUnresolved(InetSocketAddress proxyAddress) { + return proxyAddress.isUnresolved() + ? proxyAddress + : InetSocketAddress.createUnresolved(proxyAddress.getHostString(), proxyAddress.getPort()); } public String getServerName() { return serverName; } + /** + * Returns the proxy address connections should be opened to. + * + *

Unpinned, this is the stored proxy address as-is — always unresolved (see {@link + * #storeUnresolved}), which {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} + * expands to every proxy A-record, trying each in turn — so a single unreachable proxy IP no + * longer fails the connection. Re-resolving here instead would block whichever event loop called + * us, and would bypass a custom Netty resolver. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns that one proxy IP. That is what + * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} sees: it + * runs inside Netty's channel initializer, so it gets the exact IP the channel is connected to + * without a lookup on the event loop. + */ @NonNull @Override public InetSocketAddress resolve() { - try { - InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); - if (aRecords.length == 0) { - // Probably never happens, but the JDK docs don't explicitly say so - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName()); - } - // The order of the returned address is unspecified. Sort by IP to make sure we get a true - // round-robin - Arrays.sort(aRecords, IP_COMPARATOR); - int index = - (aRecords.length == 1) - ? 0 - : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; - return new InetSocketAddress(aRecords[index], proxyAddress.getPort()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName(), e); + return pinnedAddress != null ? pinnedAddress : proxyAddress; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + // Mirrors DefaultEndPoint and ClientRoutesEndPoint: an address this endpoint cannot hold in an + // InetSocketAddress field skips pinning rather than failing the connection, and so does an + // unresolved one. resolve() hands the proxy address over unresolved, and ChannelFactory passes + // it straight back when the user disabled the resolver or a custom one declines it; pinning + // that would freeze this endpoint on a name that must re-expand on every connect -- no address + // stability gained, and the proxy's A-record fallback silenced for good. + if (!(resolvedAddress instanceof InetSocketAddress) + || ((InetSocketAddress) resolvedAddress).isUnresolved() + || resolvedAddress.equals(this.pinnedAddress)) { + return this; } + return new SniEndPoint(proxyAddress, serverName, (InetSocketAddress) resolvedAddress); + } + + /** + * {@inheritDoc} + * + *

{@code true}: the proxy routes by server name, so every one of its A-records reaches this + * same node, and connections may be spread across them. That restores what this endpoint did + * itself before resolution moved to the connection layer, when {@code resolve()} sorted the proxy + * A-records and rotated through them on every call. + */ + @Override + public boolean addressesAreInterchangeable() { + return true; } @Override @@ -94,10 +173,10 @@ public int hashCode() { @Override public String toString() { - // Note that this uses the original proxy address, so if there are multiple A-records it won't - // show which one was selected. If that turns out to be a problem for debugging, we might need - // to store the result of resolve() in Connection and log that instead of the endpoint. - return proxyAddress.toString() + ":" + serverName; + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which proxy IP a given + // connection landed on is in the channel's own toString(), which Netty builds from the actual + // remote address. + return proxyAddress + ":" + serverName; } @NonNull @@ -110,10 +189,4 @@ public String asMetricPrefix() { } return hostString.replace('.', '_') + ':' + proxyAddress.getPort() + '_' + serverName; } - - @SuppressWarnings("UnnecessaryLambda") - private static final Comparator IP_COMPARATOR = - (InetAddress address1, InetAddress address2) -> - UnsignedBytes.lexicographicalComparator() - .compare(address1.getAddress(), address2.getAddress()); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index 1bb8e343d96..919fc26b8e0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -141,4 +141,42 @@ public interface TopologyMonitor extends AsyncAutoCloseable { * {@link DefaultTopologyMonitor}) should override this method. */ default void resetColumnCaches() {} + + /** + * Whether this monitor re-resolves node addresses dynamically on every connection attempt (for + * example by re-resolving a proxy hostname each time), rather than relying on an endpoint address + * captured once at node-registration time. + * + *

When this returns {@code true}, the control connection's reconnection query plan must not + * append the original contact points as a DNS re-resolution fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}): the monitor + * already keeps addresses fresh, and appending raw contact points could resurrect nodes that the + * monitor has authoritatively removed. + * + *

The default implementation returns {@code false}, which is correct for {@link + * DefaultTopologyMonitor}: the peer nodes it registers hold a {@code DefaultEndPoint} built from + * the broadcast RPC address in {@code system.peers}, an already-resolved physical IP that never + * needs re-resolving. + * + *

Unless the configured {@code AddressTranslator} hands back a name -- {@code + * SubnetAddressTranslator} does, since its {@code resolve-addresses} option defaults to {@code + * false}. Such a peer endpoint is re-expanded per connection attempt by {@code + * ChannelFactory}, and if that name maps to more than one host, one {@code Node}'s connections + * can land on different ones while routing, shard awareness and per-node metrics all attribute + * them to that single node. The candidate loop keeps them in resolver order rather than shuffling + * for an identified node (see {@code ChannelFactory#shuffleAndLimit}), so a pool stays on one + * host in practice, but the driver has no way to verify the premise. That is a property of the + * translator's output, not of this monitor, so it does not change what this flag reports. + * + *

The connected node's own {@code EndPoint} is a different case again. It originates from the + * contact point the control connection used, and {@code ChannelFactory} binds it to the single + * address that connection reached (see {@code PinnableEndPoint}), so it does not re-expand + * on later connection attempts. Recovering from an address change for that node therefore depends + * on this flag being {@code false}, i.e. on the contact-point fallback described above. + * + *

Proxy-based monitors that re-resolve per call should override this to return {@code true}. + */ + default boolean reresolvesNodeAddresses() { + return false; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java index 3d7dc50a7c0..6000510b0e9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java @@ -167,6 +167,36 @@ protected void cancelMetricsExpirationTimeout() { } } + /** + * Moves a pending expiration from the updater being replaced onto this one. See {@link + * NodeMetricUpdater#adoptExpirationFrom}. + * + *

Re-armed rather than handed over as-is, so the replacement's own {@code + * startMetricsExpirationTimeout()} runs and the countdown belongs to the object whose metrics it + * will clear. That restarts the clock; expiry is a coarse, hour-scale cleanup and the node has to + * stay down for the whole period either way, so the reset is not worth carrying the original + * deadline around for. + * + *

Only a timeout that was still pending is carried over. {@link Timeout#cancel()} returns + * {@code false} when the task has already run or was already cancelled, and the first of those is + * reachable: the task is {@code clearMetrics()} followed by {@code + * cancelMetricsExpirationTimeout()}, so an endpoint change landing between the two finds a + * non-null reference to an expired timeout. Re-arming on that would start a fresh hour-long + * countdown and expire the replacement's freshly registered metrics, even though this node's + * metrics had already been cleared once. + */ + public void adoptExpirationFrom(NodeMetricUpdater previous) { + if (!(previous instanceof AbstractMetricUpdater)) { + return; + } + Timeout pending = + ((AbstractMetricUpdater) previous).metricsExpirationTimeoutRef.getAndSet(null); + if (pending == null || !pending.cancel()) { + return; + } + startMetricsExpirationTimeout(); + } + protected Timeout newTimeout() { return context .getNettyOptions() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java index 93d003f0a03..af84782c989 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/NodeMetricUpdater.java @@ -19,4 +19,23 @@ import com.datastax.oss.driver.api.core.metrics.NodeMetric; -public interface NodeMetricUpdater extends MetricUpdater {} +public interface NodeMetricUpdater extends MetricUpdater { + + /** + * Takes over the metrics-expiration countdown from the updater this one is replacing, when a node + * rebuilds its updater after its endpoint changed. + * + *

Without this the countdown is simply lost. It is armed and cancelled through {@code + * node.getMetricUpdater()} -- by the metrics factories, on node state events -- so once a node + * has swapped in a replacement, the cancel that a later UP event triggers reaches the new updater + * and finds nothing, while the old updater's timer is still pending on an object nothing else + * refers to. Both halves of that are wrong: the replacement never expires, because a node that is + * already down will not produce another DOWN event to arm it, and the orphan eventually fires + * {@link #clearMetrics()} on names it recomputes from whatever endpoint the node holds by then. + * + *

Implementations that do not expire metrics can ignore this. + */ + default void adoptExpirationFrom(NodeMetricUpdater previous) { + // nothing to hand over + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java index 8905edb9192..7d408fb999b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.util; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; +import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -56,4 +57,36 @@ public static Set extract(String address, boolean resolve) { return result; } } + + /** + * Whether {@code address} denotes a host name, as opposed to an IP address written out in + * literal form. + * + *

The distinction matters wherever a name is treated as something that can be resolved — and + * re-resolved — while a literal is taken as the final answer. Both forms can appear resolved or + * unresolved, so neither {@link InetSocketAddress#isUnresolved()} nor the presence of an {@link + * InetAddress} tells them apart. + * + *

Performs no lookup of any kind. + * + *

One known imprecision, on the unresolved branch: {@code InetAddresses.isInetAddress} rejects + * a zone suffix, so an unresolved address built over a scoped IPv6 literal (say {@code + * fe80::1%eth0}) is reported as a name. Harmless where this is used — the string still resolves + * to exactly that address, and re-attaching it as a "host name" is a no-op — and it does not + * affect the resolved branch, where {@code getHostString()} and {@code getHostAddress()} both + * carry the zone and so compare equal. + */ + public static boolean carriesName(InetSocketAddress address) { + String hostString = address.getHostString(); + if (hostString == null) { + return false; + } + // A resolved address is compared against the literal its own bytes produce, which is cheaper + // and + // stricter than parsing; an unresolved one has no bytes, so its string has to be parsed. + InetAddress ip = address.getAddress(); + return ip != null + ? !hostString.equals(ip.getHostAddress()) + : !InetAddresses.isInetAddress(hostString); + } } diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 590784b70c5..ac1e3f39450 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -541,6 +541,23 @@ datastax-java-driver { # Overridable in a profile: no max-orphan-requests = 256 + # The maximum number of addresses a single connection attempt will try, when the endpoint it + # connects to is a DNS name that resolves to several addresses. + # + # The addresses are tried in random order, and each one tried is a full TCP connect plus + # protocol handshake -- with wrong credentials, that includes a rejected login. This cap bounds + # what one attempt can cost in time and in login attempts. Addresses beyond the cap are not + # lost: the order is shuffled again on every attempt, so successive attempts (e.g. reconnection + # rounds) sample different subsets. + # + # Setting this to 1 restores pre-multi-address behavior: one address tried per attempt. + # + # Required: yes + # Modifiable at runtime: yes, the new value will be used for connections created after the + # change. + # Overridable in a profile: no + max-candidate-addresses = 5 + # Whether to log non-fatal errors when the driver tries to open a new connection. # # This error as recoverable, as the driver will try to reconnect according to the reconnection @@ -1233,27 +1250,25 @@ datastax-java-driver { } - # Whether to resolve the addresses passed to `basic.contact-points`. + # DEPRECATED: this option no longer has any effect and will be removed in a future release. # - # If this is true, addresses are created with `InetSocketAddress(String, int)`: the host name will - # be resolved the first time, and the driver will use the resolved IP address for all subsequent - # connection attempts. + # Contact points given here are now always kept as unresolved hostnames and expanded to all of + # their DNS-mapped IPs lazily at connection time. This means the driver tries every IP a hostname + # resolves to, and re-resolves the hostname on each new connection so DNS changes are picked up + # automatically. Previously this option selected between resolving a contact-point hostname once + # (true) and re-resolving it on every connection (false); that distinction no longer applies. # - # If this is false, addresses are created with `InetSocketAddress.createUnresolved()`: the host - # name will be resolved again every time the driver opens a new connection. This is useful for - # containerized environments where DNS records are more likely to change over time (note that the - # JVM and OS have their own DNS caching mechanisms, so you might need additional configuration - # beyond the driver). + # The lookup goes through Netty's configured AddressResolverGroup -- the same resolver an + # unresolved address would have reached had it been passed straight to Bootstrap.connect() -- so a + # custom resolver installed via NettyOptions.afterBootstrapInitialized() still applies. With + # Netty's default (JDK) resolver the lookup blocks the I/O event loop it runs on; install + # DnsAddressResolverGroup if you need it to be non-blocking. # - # This option only applies to the contact points specified in the configuration. It has no effect - # on: - # - programmatic contact points passed to SessionBuilder.addContactPoints: these addresses are - # built outside of the driver, so it is your responsibility to provide unresolved instances. - # - dynamically discovered peers: the driver relies on Cassandra system tables, which expose raw - # IP addresses. Use a custom address translator to convert them to unresolved addresses (if - # you're in a containerized environment, you probably already need address translation anyway). + # This option only ever applied to the contact points specified in the configuration -- never to + # programmatic contact points passed to SessionBuilder.addContactPoints, nor to dynamically + # discovered peers. # - # Required: no (defaults to false) + # Required: no # Modifiable at runtime: no # Overridable in a profile: no advanced.resolve-contact-points = false @@ -2347,14 +2362,30 @@ datastax-java-driver { } reconnection { - # Whether to forcibly add original contact points held by MetadataManager to the reconnection plan, - # in case there is no live nodes available according to LBP. - # Experimental. + # Whether to append the original contact points held by MetadataManager to the reconnection + # plan, after the live nodes reported by the load balancing policy. + # + # This is also the driver's DNS re-resolution path. Contact points are kept as unresolved + # hostnames and expanded to their current DNS IPs at connection time, through Netty's + # configured resolver. Metadata nodes, in contrast, store an already-resolved endpoint that + # is never re-resolved, so once DNS records change they would otherwise become stale. Keeping + # this enabled lets control-connection reconnects re-resolve the original hostnames and pick up + # the new IPs once the live-node plan is exhausted. + # + # Note the cost: the contact points are appended without being compared against the live-node + # plan, because at plan time they are still hostnames and the live nodes are already-resolved + # IPs. When DNS has not changed they therefore expand to addresses the plan just failed on, so + # a reconnection round that exhausts the live nodes retries roughly twice as many addresses. + # Each of those costs a connect attempt -- up to `advanced.connection.connect-timeout` -- and, + # if the address accepts the connection but then stalls, the init handshake on top, whose + # steps each arm their own `advanced.connection.init-query-timeout`. Set this to false if you + # do not need DNS re-resolution of contact points and would rather keep reconnection rounds + # short. # # Required: yes # Modifiable at runtime: yes, the new value will be used for checks issued after the change. # Overridable in a profile: no - fallback-to-original-contact-points = false + fallback-to-original-contact-points = true } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java new file mode 100644 index 00000000000..dbe60ae4432 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.context.NettyOptions; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelInboundHandlerAdapter; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies the {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} contract: the hook runs on + * a handler-less bootstrap, and a handler it installs is replaced by the driver's own. + */ +public class ChannelFactoryBootstrapHookTest extends ChannelFactoryTestBase { + + @Test + public void should_replace_handler_installed_by_bootstrap_hook() { + // Given – a hook that (incorrectly) installs its own channel handler. The driver sets its own + // handler on each per-attempt copy afterwards, logging a one-time warning; if the dummy + // handler below survived instead, the protocol handshake would never happen and this connect + // would fail on the init timeout. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.handler(new ChannelInboundHandlerAdapter()); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java new file mode 100644 index 00000000000..383eda9219d --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryConnectHookTest.java @@ -0,0 +1,549 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThat; +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.connection.ConnectionInitException; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.ProtocolConstants; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Register; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Error; +import com.datastax.oss.protocol.internal.response.Ready; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +/** + * Verifies the two steps {@link ChannelFactory} runs between protocol initialization and the + * completion of a candidate attempt: the caller's {@link ConnectHook}, and the REGISTER request + * that moved out of the init handshake so that a channel the hook is about to reject never + * registers for events. + */ +public class ChannelFactoryConnectHookTest extends ChannelFactoryTestBase { + + private static final Duration HOOK_TIMEOUT = Duration.ofSeconds(5); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + private void givenNegotiableProtocol() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + } + + /** + * Drives one candidate's handshake to successful completion. Only the factory's first channel + * sends OPTIONS, so later candidates start straight at STARTUP. + */ + private void completeInit() { + Frame requestFrame = readOutboundFrame(); + if (requestFrame.message instanceof Options) { + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + } + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("mockClusterName")); + } + + private static DriverChannelOptions optionsWithHook(ConnectHook hook) { + return DriverChannelOptions.builder().withConnectHook(hook, HOOK_TIMEOUT).build(); + } + + private static DriverChannelOptions optionsWithHookAndEvents(ConnectHook hook) { + return DriverChannelOptions.builder() + .withConnectHook(hook, HOOK_TIMEOUT) + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + } + + @Test + public void should_complete_candidate_only_after_hook_accepts() { + // Given — a hook whose completion the test controls. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + List vettedChannels = new CopyOnWriteArrayList<>(); + ConnectHook hook = + channel -> { + vettedChannels.add(channel); + return gate; + }; + + // When — init completes but the hook has not answered yet. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt is not successful until the hook says so. (The hook runs on the event + // loop after init succeeds, so wait for the invocation before asserting on the future.) + await().atMost(java.time.Duration.ofSeconds(2)).until(() -> vettedChannels.size() == 1); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + + // When + gate.complete(null); + + // Then — the vetted channel is the one handed to the caller. + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel).isSameAs(vettedChannels.get(0))); + } + + @Test + public void should_try_next_address_when_hook_rejects_candidate() { + // Given — a name expanding to two addresses of the live server, and a hook that rejects the + // first candidate and accepts the second: the caller's acceptance criteria are per address, + // and a rejection must not write off the endpoint. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + AtomicInteger invocations = new AtomicInteger(); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + if (invocations.incrementAndGet() == 1) { + result.completeExceptionally(new IllegalStateException("not this one")); + } else { + result.complete(null); + } + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + optionsWithHook(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + completeInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + assertThat(invocations.get()).isEqualTo(2); + } + + @Test + public void should_fail_connect_when_hook_rejects_the_last_candidate() { + // Given — a single address, so the rejection has nowhere to advance to. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException rejection = new IllegalStateException("cannot identify itself"); + ConnectHook hook = + channel -> { + CompletableFuture result = new CompletableFuture<>(); + result.completeExceptionally(rejection); + return result; + }; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the rejection's cause is preserved for diagnosis. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(rejection); + }); + } + + @Test + public void should_treat_synchronous_hook_throw_as_rejection() { + // A hook is a caller-supplied callback running inside a Netty listener: a leaked throwable + // would otherwise leave the attempt hanging forever. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + IllegalStateException thrown = new IllegalStateException("hook blew up"); + ConnectHook hook = + channel -> { + throw thrown; + }; + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).isSameAs(thrown); + }); + } + + @Test + public void should_reject_candidate_when_hook_times_out() { + // Given — a hook whose stage never completes; only the driver can bound that. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = channel -> new CompletableFuture<>(); + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ofMillis(100)).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("timed out")); + } + + @Test + public void should_register_for_events_only_after_hook_accepts() { + // Given — events requested and a gated hook. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHookAndEvents(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + // The hook has not accepted yet: were REGISTER part of init, it would already be on the wire. + assertThat(tryReadOutboundFrame(200)).isNull(); + gate.complete(null); + + // Then — REGISTER goes out only now, and the attempt completes once it is acknowledged. + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + assertThat(((Register) registerFrame.message).eventTypes).containsExactly("foo", "bar"); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_fail_candidate_when_the_step_after_the_hook_throws() + throws InterruptedException { + // Given — events requested, and a config read that starts throwing once the hook has accepted + // (an option gone missing across a config reload, a custom DriverConfig, a wrong type). + // registerForEvents() runs inside the hook stage's whenComplete callback: nobody consumes the + // stage that callback returns, and the hook timeout that would have failed the candidate has + // just been cancelled, so without a blanket catch there the attempt would hang forever. + givenNegotiableProtocol(); + AtomicBoolean poisoned = new AtomicBoolean(); + when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT)) + .thenAnswer( + invocation -> { + if (poisoned.get()) { + throw new IllegalStateException("config reloaded without the option"); + } + return Duration.ofMillis(500); + }); + ChannelFactory factory = newChannelFactory(); + AtomicReference vettedChannel = new AtomicReference<>(); + ConnectHook hook = + channel -> { + vettedChannel.set(channel); + poisoned.set(true); + return CompletableFuture.completedFuture(null); + }; + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHookAndEvents(hook), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the attempt fails instead of hanging, REGISTER never goes out, and the channel it had + // already opened is closed rather than left dangling with nothing holding it. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error.getCause()).hasMessageContaining("config reloaded"); + }); + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(vettedChannel.get().closeFuture().await(500, TimeUnit.MILLISECONDS)) + .as("the abandoned candidate's channel should have been closed") + .isTrue(); + } + + @Test + public void should_register_for_events_after_init_when_no_hook_is_set() { + // Given — events but no hook (REGISTER still has to happen even when there is nothing to vet). + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_try_next_address_when_registration_fails() { + // Given — two addresses; the first candidate's REGISTER is refused. A registration failure is + // a per-candidate failure, exactly as it was when REGISTER was an init step. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then — the loop advances; the second candidate registers successfully. + completeInit(); + registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Ready()); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_treat_a_zero_hook_timeout_as_unbounded() { + // The hook timeout comes from advanced.control-connection.timeout, and every other consumer of + // a + // driver timeout option reads a non-positive duration as "no timeout" (see + // AdminRequestHandler#onWriteComplete). Scheduled anyway, a zero delay fires on the next + // event-loop turn -- before any round trip can complete -- and would abandon every candidate of + // every contact point, so an operator who disabled that timeout could not initialize a session. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + CompletableFuture gate = new CompletableFuture<>(); + ConnectHook hook = channel -> gate; + DriverChannelOptions options = + DriverChannelOptions.builder().withConnectHook(hook, Duration.ZERO).build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then — the hook is given as long as it needs, and the candidate survives. + assertThat(tryReadOutboundFrame(200)).isNull(); + assertThat(channelFuture.toCompletableFuture()).isNotDone(); + gate.complete(null); + + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_the_hook_rejects() { + // The protocol version and cluster name used to be latched as soon as the transport connect and + // init handshake succeeded, which was safe while init had the last word on a candidate. It no + // longer does: this hook rejects one, and REGISTER (below) can too. A stale DNS record pointing + // at a foreign cluster would otherwise leave that cluster's name latched here, and every later + // connection -- to any node -- would fail its cluster-name check, which ChannelPool turns into + // an irreversible forced-down node. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + ConnectHook hook = + channel -> { + CompletableFuture rejected = new CompletableFuture<>(); + rejected.completeExceptionally(new IllegalStateException("no host_id")); + return rejected; + }; + + // When — the only candidate is rejected after a fully successful handshake. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, null, null, optionsWithHook(hook), NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + // Then + assertThatStage(channelFuture) + .isFailed(error -> assertThat(error).hasMessageContaining("hook")); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_not_latch_negotiated_state_from_a_candidate_whose_registration_fails() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents(ImmutableList.of("foo", "bar"), mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame(registerFrame, new Error(ProtocolConstants.ErrorCode.SERVER_ERROR, "nope")); + + // Then + assertThatStage(channelFuture).isFailed(); + assertThat(factory.protocolVersion).isNull(); + assertThat(factory.getClusterName()).isNull(); + } + + @Test + public void should_latch_negotiated_state_once_a_candidate_is_accepted() { + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + optionsWithHook(channel -> CompletableFuture.completedFuture(null)), + NoopNodeMetricUpdater.INSTANCE); + completeInit(); + + assertThatStage(channelFuture).isSuccess(); + assertThat(factory.protocolVersion).isEqualTo(DefaultProtocolVersion.V4); + assertThat(factory.getClusterName()).isEqualTo("mockClusterName"); + } + + @Test + public void should_stop_the_candidate_loop_when_the_server_does_not_know_the_event_type() { + // Given — two addresses, and a server that does not support the event type being registered + // for. Unlike every other REGISTER failure this one is node-wide: it describes the software + // running there, not the record that reached it, so replaying it against the remaining + // addresses can only fail the same way while paying a full TCP connect plus the + // STARTUP/AUTH/cluster-name handshake for each. Stopping at the first restores what this + // rejection cost while REGISTER was an init step, which was one failed connect per contact + // point. + givenNegotiableProtocol(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + // Then — the attempt is already failed, without the second address having been dialled. Compare + // should_try_next_address_when_registration_fails, where the stage is still pending at this + // point because the loop moved on. + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + }); + } + + @Test + public void should_translate_client_routes_register_rejection() { + // The one REGISTER rejection with a known cause keeps its clear message, as it had when + // REGISTER was an init step: the caller (ClientRoutesTopologyMonitor.init()) reports it + // instead of silently degrading. + givenNegotiableProtocol(); + ChannelFactory factory = newChannelFactory(); + DriverChannelOptions options = + DriverChannelOptions.builder() + .withEvents( + ImmutableList.of(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE), + mock(EventCallback.class)) + .build(); + + CompletionStage channelFuture = + factory.connect(SERVER_ADDRESS, null, null, options, NoopNodeMetricUpdater.INSTANCE); + completeInit(); + Frame registerFrame = readOutboundFrame(); + assertThat(registerFrame.message).isInstanceOf(Register.class); + writeInboundFrame( + registerFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, + "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); + + assertThatStage(channelFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(ConnectionInitException.class); + assertThat(error).hasMessageContaining("CLIENT_ROUTES_CHANGE"); + assertThat(error).hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); + }); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java new file mode 100644 index 00000000000..d62d9bc9d58 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -0,0 +1,816 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.util.AddressUtils; +import com.datastax.oss.protocol.internal.Frame; +import com.datastax.oss.protocol.internal.request.Options; +import com.datastax.oss.protocol.internal.request.Startup; +import com.datastax.oss.protocol.internal.response.Authenticate; +import com.datastax.oss.protocol.internal.response.Ready; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.channel.local.LocalAddress; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketAddress; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.Test; + +/** + * Verifies how {@link ChannelFactory#connect} treats the several addresses a name expands to: they + * are tried in sequence in a shuffled order, at most {@code + * advanced.connection.max-candidate-addresses} of them, and failures are aggregated rather than + * dropped. + * + *

The expansion itself is exercised in {@link ChannelFactoryNettyResolverTest}; here the + * resolver is only the mechanism for producing more than one address from a single endpoint. + */ +public class ChannelFactoryMultiAddressTest extends ChannelFactoryTestBase { + + // Local addresses that no server is bound to: connecting to them fails immediately. + private static final SocketAddress UNREACHABLE_1 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-1"); + private static final SocketAddress UNREACHABLE_2 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-2"); + private static final SocketAddress UNREACHABLE_3 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-3"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_fail_with_suppressed_causes_when_all_addresses_are_unreachable() { + // Given – a name that expands to two dead addresses. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the future fails, and the earlier address's failure is preserved as a suppressed + // exception on the last one's error rather than being silently dropped. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e.getSuppressed()) + .as("earlier address failures should be attached as suppressed exceptions") + .isNotEmpty()); + } + + @Test + public void should_attach_each_earlier_failure_at_most_once() { + // Given – three dead addresses, so there are earlier failures to carry. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – attaching to the error being reported mutates an object the driver does not own, so + // every cause has to appear at most once and the error must never suppress itself. Nothing + // stops two candidates from failing with the same instance -- a pipeline handler that throws a + // stackless singleton, say -- and such an instance would otherwise grow a suppressed entry on + // every connect for as long as the JVM lives. + assertThatStage(channelFuture) + .isFailed( + e -> { + List suppressed = Arrays.asList(e.getSuppressed()); + assertThat(suppressed).isNotEmpty(); + for (int i = 0; i < suppressed.size(); i++) { + assertThat(suppressed.get(i)).isNotSameAs(e); + for (int j = i + 1; j < suppressed.size(); j++) { + assertThat(suppressed.get(i)).isNotSameAs(suppressed.get(j)); + } + } + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_a_contact_point() { + // Given – a name expanding to two addresses, both the same live server, which asks for + // authentication the driver has no provider for. The endpoint is a bare contact point, so the + // driver does not yet know which node -- or even which cluster -- any of these addresses is. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – the loop advances. Authentication completes before the cluster-name check + // (ProtocolInitHandler runs STARTUP -> AUTH_RESPONSE -> GET_CLUSTER_NAME), so a stale record + // pointing at a foreign cluster that wants different credentials fails here rather than at the + // cluster-name mismatch that would have advanced. Writing off the whole name on this error + // would therefore make that rule unreachable in exactly the multi-record case this loop is for. + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message) + .as("the second candidate should have been attempted") + .isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // And – once both are exhausted the failure is still an AuthenticationException, with the first + // address's copy attached rather than dropped. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + @Test + public void should_surface_the_authentication_failure_when_another_address_fails_on_transport() { + // Given – a name expanding to the live server (which asks for authentication the driver has no + // provider for) and a dead address. The shuffled order does not matter: whichever is tried + // first, the pass ends with one authentication failure and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, UNREACHABLE_1))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + + // Then – even when the transport failure is the *last* error, propagating it would report a + // connect failure for what is really a rejected password: callers branch on the type of what + // they receive (ChannelPool#handleError, ControlConnection's auth-specific warning and its + // errors.connection.auth metric), and with a shuffled multi-record name which address happens + // to be tried last is arbitrary. The classified failure wins, and the transport one is still + // attached. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("an authentication failure must not be demoted by a later transport failure") + .isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the transport failure should still be attached") + .hasSize(1); + assertThat(e.getSuppressed()[0]).isNotInstanceOf(AuthenticationException.class); + }); + } + + @Test + public void should_not_surface_a_cluster_name_mismatch_that_only_one_address_reported() { + // Given – a factory that already knows the cluster name (from a first connection), then a name + // expanding to the live server -- which now answers with a *different* cluster name -- and a + // dead address. Whichever order the shuffle picks, the pass ends with one cluster-name mismatch + // and one transport failure. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + CompletionStage firstChannel = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + assertThatStage(firstChannel).isSuccess(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(SERVER_ADDRESS.resolve(), UNREACHABLE_1))); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // The dead address sends nothing, so this drives the live candidate whichever position it got. + // The protocol version and the product type are known by now, hence no OPTIONS request. + writeInboundFrame(readOutboundFrame(), new Ready()); + writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("wrongClusterName")); + + // Then – the mismatch must not be the failure that surfaces, not even as the last error of the + // pass. ChannelPool#handleError turns it into TopologyEvent.forceDown and nothing in the driver + // ever reverses one, while one address of a multi-record name fronting another cluster is a + // stale record rather than a verdict about the node. It is still attached, so nothing is lost. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e) + .as("a mismatch from a single address must not be promoted over the others") + .isNotInstanceOf(ClusterNameMismatchException.class); + assertThat( + Arrays.stream(e.getSuppressed()) + .anyMatch(s -> s instanceof ClusterNameMismatchException)) + .as("the mismatch should still be attached as a suppressed exception") + .isTrue(); + }); + } + + @Test + public void should_try_next_address_when_authentication_fails_on_an_identified_node() { + // Given – the same server and the same two addresses, but a node the driver has already + // identified (nodeIsIdentified = true, i.e. its host id was read from system.local/peers). + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(context.getAuthProvider()).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + failAuthenticationOnNextCandidate(); + + // Then – the loop advances, exactly as it does for a contact point: no single address's + // failure writes off the endpoint, and the candidate cap -- not a node-wide classification -- + // is what bounds the cost of genuinely wrong credentials. + failAuthenticationOnNextCandidate(); + + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(AuthenticationException.class); + assertThat(e.getSuppressed()) + .as("the first candidate's failure should be attached as suppressed") + .hasSize(1); + }); + } + + /** Drives one candidate's handshake as far as the server's authentication challenge. */ + private void failAuthenticationOnNextCandidate() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(requestFrame, new Authenticate("mockAuthenticator")); + } + + @Test + public void should_stop_after_the_configured_number_of_addresses() { + // Given – a name expanding to three addresses, but a cap of two. Every address tried is a full + // connect plus handshake -- and, with wrong credentials, a rejected login -- and the + // reconnection fallback re-appends the contact points to every round, so an unbounded walk + // would repeat per contact point, per round, for as long as the session lives. The cap is what + // bounds that. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – only two addresses were dialed. The count of suppressed entries is not what to assert + // on: a single dead candidate can contribute two entangled failures (the transport refusal, + // plus an init-write failure that PromiseCombiner attaches to it as suppressed). The set of + // addresses named anywhere in the aggregate is what reflects the dials. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(mentionedUnreachableAddresses(e)) + .as("a cap of 2 means exactly two addresses dialed") + .hasSize(2)); + } + + private static final Pattern UNREACHABLE_NAME = Pattern.compile("unreachable-\\d"); + + /** The distinct dead-address names mentioned anywhere in {@code error}'s suppressed tree. */ + private static Set mentionedUnreachableAddresses(Throwable error) { + Set names = new HashSet<>(); + Deque toVisit = new ArrayDeque<>(); + toVisit.push(error); + while (!toVisit.isEmpty()) { + Throwable current = toVisit.pop(); + String message = current.getMessage(); + if (message != null) { + Matcher matcher = UNREACHABLE_NAME.matcher(message); + while (matcher.find()) { + names.add(matcher.group()); + } + } + for (Throwable suppressed : current.getSuppressed()) { + toVisit.push(suppressed); + } + } + return names; + } + + @Test + public void should_shuffle_candidates_without_losing_any() { + // The order is random per connect -- that is what spreads load across a name's records and + // varies the starting address between successive attempts -- but every address must survive + // the shuffle, since the loop's fallback walks this list. + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), true)) + .containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + } + + @Test + public void should_order_candidates_by_the_injected_random_source() { + // The injection point for ordering-sensitive tests: a seeded Random produces the same + // permutation on two factories, so a scenario that needs a particular order picks a seed + // instead of depending on a sort the production code no longer performs. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory one = newChannelFactory(); + ChannelFactory other = newChannelFactory(); + one.random = new Random(42); + other.random = new Random(42); + + assertThat(one.shuffleAndLimit(addresses, true)) + .containsExactlyElementsOf(other.shuffleAndLimit(addresses, true)); + } + + @Test + public void should_leave_a_single_address_alone() { + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Collections.singletonList(UNREACHABLE_1), true)) + .containsExactly(UNREACHABLE_1); + } + + @Test + public void should_truncate_the_shuffled_list_to_the_cap() { + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + + List capped = factory.shuffleAndLimit(addresses, true); + + assertThat(capped).hasSize(2); + assertThat(addresses).containsAll(capped); + } + + @Test + public void should_clamp_the_cap_to_at_least_one_address() { + // Zero or a negative value cannot mean "dial nothing" -- the attempt would fail without ever + // trying an address. It degrades to the pre-multi-address behavior of one address per attempt. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(0); + ChannelFactory factory = newChannelFactory(); + + assertThat(factory.shuffleAndLimit(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2), true)) + .hasSize(1); + } + + @Test + public void should_not_shuffle_when_the_addresses_are_not_interchangeable() { + // A name that may denote different hosts -- what an AddressTranslator can hand back, and + // SubnetAddressTranslator does by default -- must keep the resolver's order: a random one would + // scatter a single Node's pool across hosts that routing, shard awareness and per-node metrics + // all attribute to that one node. Keeping the order makes such a pool converge on one address, + // as it did before multi-address support, while the rest of the list still serves as fallback. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3); + ChannelFactory factory = newChannelFactory(); + // A seed that does permute this list, so the assertion below fails if the shuffle still runs. + factory.random = new Random(42); + assertThat(factory.shuffleAndLimit(addresses, true)).isNotEqualTo(addresses); + + assertThat(factory.shuffleAndLimit(addresses, false)).containsExactlyElementsOf(addresses); + } + + @Test + public void should_still_cap_the_candidates_when_the_order_is_kept() { + // The cap is what bounds the cost of one attempt, and that applies whether or not the addresses + // were shuffled. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(2); + ChannelFactory factory = newChannelFactory(); + + assertThat( + factory.shuffleAndLimit( + Arrays.asList(UNREACHABLE_1, UNREACHABLE_2, UNREACHABLE_3), false)) + .containsExactly(UNREACHABLE_1, UNREACHABLE_2); + } + + // ---- spreadAcrossAddresses() ---------------------------------------------- + + /** The endpoints below only have to exist; nothing in this section connects to them. */ + private static final InetSocketAddress SOME_ADDRESS = + InetSocketAddress.createUnresolved("node.example.com", 9042); + + @Test + public void should_spread_across_the_addresses_of_a_contact_point() { + // Nothing is known about a contact point's addresses -- they may be different nodes -- so there + // is no node identity to preserve and every shape is spread. + assertThat(ChannelFactory.spreadAcrossAddresses(new DefaultEndPoint(SOME_ADDRESS), false)) + .isTrue(); + assertThat( + ChannelFactory.spreadAcrossAddresses( + new SniEndPoint(SOME_ADDRESS, "server-name"), false)) + .isTrue(); + } + + @Test + public void should_spread_across_the_addresses_of_an_identified_node_behind_a_proxy() { + // An SNI proxy routes by server name, so every one of its A-records reaches this same node. + // Spreading is what SniEndPoint#resolve() itself did, by rotating through the sorted records, + // before resolution moved to the connection layer. + assertThat( + ChannelFactory.spreadAcrossAddresses( + new SniEndPoint(SOME_ADDRESS, "server-name"), true)) + .isTrue(); + } + + @Test + public void should_keep_the_order_for_an_identified_node_on_a_plain_endpoint() { + // The case the interchangeability flag exists to exclude: a DefaultEndPoint holding a name an + // AddressTranslator supplied carries no guarantee that its addresses are one node. + assertThat(ChannelFactory.spreadAcrossAddresses(new DefaultEndPoint(SOME_ADDRESS), true)) + .isFalse(); + } + + @Test + public void should_keep_the_order_for_an_identified_node_on_a_third_party_endpoint() { + // An EndPoint that does not implement PinnableEndPoint cannot say, and the conservative reading + // is the one that preserves node identity. + EndPoint thirdParty = mock(EndPoint.class); + when(thirdParty.resolve()).thenReturn(SOME_ADDRESS); + + assertThat(ChannelFactory.spreadAcrossAddresses(thirdParty, true)).isFalse(); + } + + // ---- reattachHostname() --------------------------------------------------- + + @Test + public void should_reattach_queried_hostname_to_nameless_resolved_address() throws Exception { + // A custom resolver may build its results from raw address bytes; the queried name must be + // re-attached so TLS hostname validation checks the configured name (not the IP or a PTR + // record) and reading the host name never triggers a reverse lookup on the event loop. + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9999); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.isUnresolved()).isFalse(); + // getHostString() never looks anything up; getHostName() reverse-resolves a *nameless* + // address, so it returning the queried name proves the name is embedded, not looked up. + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getHostName()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + // The candidate's port wins over the original's: a resolver may remap ports too. + assertThat(result.getPort()).isEqualTo(9999); + // Equality is unchanged (a resolved InetSocketAddress compares IP bytes + port only), so + // pinning and the pin-equality shortcuts behave exactly as with the raw candidate. + assertThat(result).isEqualTo(candidate); + } + + @Test + public void should_override_resolver_provided_hostname_with_queried_name() throws Exception { + // A resolver may label its results with a canonical/CNAME name of its own. That name would end + // up on the pinned endpoint and hence be the one TLS hostname verification checks the server + // certificate against, so the name the user configured has to win over it. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("cname.example.fake", new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_pass_candidate_through_when_it_already_carries_the_queried_name() + throws Exception { + // The common case: the JDK and Netty-DNS resolvers attach the queried name themselves, so + // there is nothing to rebuild. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("test.cluster.fake", new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + } + + @Test + public void should_pass_non_inet_candidate_through() { + // The local-transport addresses these unit tests connect over must never be touched. + assertThat(ChannelFactory.reattachHostname(HOSTNAME, UNREACHABLE_1)).isSameAs(UNREACHABLE_1); + } + + @Test + public void should_pass_candidate_through_when_original_carries_no_name() throws Exception { + // An original written as an IP literal has no name to carry over, and inventing one from the + // literal would be worse than leaving the candidate alone: a resolver is free to redirect it to + // a different IP, which would then be labelled with the literal form of a *different* address. + InetSocketAddress original = new InetSocketAddress("127.0.0.1", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + assertThat(AddressUtils.carriesName(original)).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) + .isFalse(); + } + + @Test + public void should_reattach_name_of_a_resolved_original() throws Exception { + // A resolved original still reaches the resolver -- whether an address needs resolving is the + // resolver's call, and a custom one may redirect it. Its name is the one the operator + // configured, so it must survive onto whatever the resolver substitutes, exactly as it did when + // Netty resolved the TCP destination and the channel kept the original endpoint for TLS. + InetSocketAddress original = new InetSocketAddress("localhost", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(AddressUtils.carriesName(original)).isTrue(); + assertThat(result.getHostString()).isEqualTo("localhost"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + } + + @Test + public void should_reattach_hostname_to_nameless_ipv6_address() throws Exception { + byte[] loopback = new byte[16]; + loopback[15] = 1; // ::1 + InetSocketAddress candidate = new InetSocketAddress(InetAddress.getByAddress(loopback), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isEqualTo(candidate.getAddress()); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_scope_when_reattaching_to_a_scoped_ipv6_address() throws Exception { + // A link-local address only points anywhere together with its zone, so the queried name has to + // be re-attached without dropping the scope. InetAddress.getByAddress(host, bytes) cannot carry + // one, but Inet6Address.getByAddress(host, bytes, scopeId) can. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isInstanceOf(Inet6Address.class); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(3); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_zone_of_an_interface_scoped_ipv6_address() throws Exception { + // An address built from a NetworkInterface rather than from an index must keep pointing into + // the + // same zone. The numeric scope the JDK derived at construction is what the connect goes on, so + // carrying that over is enough; only the interface name, a toString() detail, is not. + Inet6Address linkLocal = firstInterfaceScopedIpv6Address(); + assumeThat(linkLocal).as("no interface-scoped IPv6 address on this host").isNotNull(); + InetSocketAddress candidate = new InetSocketAddress(linkLocal, 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(linkLocal.getScopeId()); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal.getAddress()); + } + + /** An interface-scoped IPv6 address of this host, or null if it has none. */ + private static Inet6Address firstInterfaceScopedIpv6Address() throws Exception { + for (NetworkInterface nif : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress address : Collections.list(nif.getInetAddresses())) { + if (address instanceof Inet6Address + && ((Inet6Address) address).getScopedInterface() != null) { + return (Inet6Address) address; + } + } + } + return null; + } + + @Test + public void should_fail_future_when_endpoint_resolve_throws() { + // ChannelFactory calls EndPoint.resolve() directly on the caller thread, so a third-party + // implementation that throws must surface as a failed future rather than an escaping exception. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + IllegalStateException failure = new IllegalStateException("resolve() blew up"); + + CompletionStage channelFuture = + factory.connect( + new ThrowingEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + @Test + public void should_fail_future_when_endpoint_resolve_returns_null() { + // EndPoint.resolve() is contractually non-null, but a broken third-party implementation must + // fail fast rather than NPE later inside an event-loop task, which would leave the future + // hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new NullResolvingEndPoint(), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("returned null")); + } + + @Test + public void should_fail_future_when_event_loop_group_is_rejecting_tasks() + throws InterruptedException { + // Resolution is dispatched to an I/O event loop; if the group is already shutting down, that + // dispatch is rejected synchronously. The rejection must fail the future rather than escape to + // the caller (connect() never used to throw) or leave the future hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + clientGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).sync(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).isInstanceOf(RejectedExecutionException.class)); + } + + /** An endpoint whose {@link EndPoint#resolve()} throws, standing in for a broken third party. */ + private static class ThrowingEndPoint implements EndPoint { + + private final RuntimeException failure; + + ThrowingEndPoint(RuntimeException failure) { + this.failure = failure; + } + + @NonNull + @Override + public SocketAddress resolve() { + throw failure; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } + + /** A broken third-party endpoint that violates {@code resolve()}'s non-null contract. */ + private static class NullResolvingEndPoint implements EndPoint { + + @NonNull + @Override + @SuppressWarnings("NullAway") // deliberately broken, that is the point of the test + public SocketAddress resolve() { + return null; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java new file mode 100644 index 00000000000..540962b4aeb --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.DefaultEventLoopGroup; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory} expands unresolved candidate addresses through Netty's + * configured {@link AddressResolverGroup}, rather than doing its own JVM DNS lookup. + * + *

This is what keeps a custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)} + * effective: before multi-address support, an unresolved address was handed straight to {@code + * Bootstrap.connect()} and Netty's resolver expanded it, so resolving anywhere else would silently + * bypass the user's configuration. + */ +public class ChannelFactoryNettyResolverTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryNettyResolverTest.class.getSimpleName() + "-unreachable"); + + /** The hostname the endpoint reports, and that only the custom resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + /** What a resolver must never hand back from {@code resolveAll}, but might. */ + private static final InetSocketAddress STILL_UNRESOLVED = + InetSocketAddress.createUnresolved("still.unresolved.fake", 9042); + + @Test + public void should_expand_unresolved_address_through_the_custom_netty_resolver() { + // Given – a resolver that maps the hostname to an unreachable address followed by the running + // local server, mimicking a DNS round-robin entry whose first record is dead. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint itself performs no resolution at all; it just yields the hostname. + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second address. + completeSimpleChannelInit(); + + // Then – the custom resolver was consulted for the hostname, and *all* the addresses it + // returned + // were tried, so the connection survived the dead first record. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the custom Netty resolver must be the one expanding the hostname") + .containsExactly(HOSTNAME); + } + + @Test + public void should_fail_when_the_custom_resolver_cannot_resolve_the_only_candidate() { + // Given – a resolver that fails every lookup. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = new TestAddressResolverGroup(null); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – no candidate survived resolution, so the connect fails with the resolver's own cause + // rather than, say, an empty-candidate-list error. + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).hasMessageContaining("mock resolver failure")); + } + + @Test + public void should_fail_with_a_diagnosable_error_when_every_expanded_address_is_unresolved() { + // Given – a resolver that "expands" the hostname to another unresolved address. A redirecting + // resolver can do this by rewriting the host without resolving it, and nothing downstream will + // resolve it either: connectToAddress() uses a bootstrap clone with disableResolver(), so Netty + // would raise UnresolvedAddressException from inside doConnect, naming neither the address nor + // the reason nothing resolved it -- for every connect of the whole session. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Collections.singletonList(STILL_UNRESOLVED))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – the failure says which endpoint and which resolver produced it, as the pass-through + // paths already did (see ChannelFactory#unusableWithoutResolution). + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(IllegalStateException.class); + assertThat(e.getMessage()).contains("test.cluster.fake"); + assertThat(e.getMessage()).contains("TestAddressResolverGroup"); + assertThat(e.getMessage()).contains("unresolved"); + }); + } + + @Test + public void should_drop_an_unresolved_expanded_address_before_applying_the_cap() { + // Given – the same resolver answering with one unusable address and one live one, and a cap of + // a + // single candidate. An address that cannot be connected to must not consume a slot in that cap: + // dropped after the truncation instead, it would leave this connect with nothing to dial. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(1); + installResolver( + new TestAddressResolverGroup(Arrays.asList(STILL_UNRESOLVED, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // Keeping the resolver's order is what makes this an assertion about the cap rather than about + // luck: the unusable address is the one the truncation would otherwise have kept. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { + // Given – Bootstrap.disableResolver() means config().resolver() is null. ChannelFactory must + // treat that as "pass the candidates through" instead of dereferencing the missing group. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(resolverGroup).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint yields an already-usable address. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – connection succeeds and the resolver was never even instantiated, let alone consulted. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.resolverRequested).isFalse(); + assertThat(resolverGroup.queried).isEmpty(); + } + + @Test + public void should_pass_already_resolved_address_through_untouched() { + // Given – an endpoint whose address is already resolved, which is the common case: metadata + // nodes hold resolved addresses from the peers rows, so this is every pool refill and every + // reconnect. A resolver with the usual semantics reports it as resolved and there is nothing + // to expand. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – no lookup was performed: had one been, it would have redirected us to UNREACHABLE and + // the connection would have failed. The decision was the resolver's own, though -- see + // should_let_the_resolver_redirect_an_already_resolved_address. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried).isEmpty(); + assertThat(resolverGroup.resolverRequested) + .as("whether an address needs resolving must be the resolver's decision") + .isTrue(); + } + + @Test + public void should_let_the_resolver_redirect_an_already_resolved_address() { + // Given – a resolver that reports even an address carrying an IP as still needing resolution, + // and redirects it. Netty consulted the resolver for every connect, resolved address or not + // (Bootstrap#doResolveAndConnect0 calls isSupported()/isResolved() on it rather than testing + // the address itself), so short-circuiting on InetSocketAddress#isUnresolved() here would take + // that away for every connect to an already-resolved node -- which is nearly all of them. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Collections.singletonList(SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint holds a resolved address that nothing is listening on. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(resolved), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the connect landed on the address the resolver substituted, which it could only do by + // having been asked about an address that already carried an IP. Exactly one lookup: the + // per-attempt bootstrap has the resolver disabled, so the substitute is connected to as-is + // rather than being handed back to the resolver (see the next test for why that matters). + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the resolver must get a say on an address that already carries an IP") + .containsExactly(resolved); + } + + @Test + public void should_try_every_candidate_when_the_resolver_redirects() { + // Given – the same redirecting resolver as above, but answering with more than one address: + // a dead one first, then the running local server. + // + // The per-attempt bootstrap must not re-resolve. Bootstrap.clone() carries the resolver + // configuration over, and Netty's own pass calls resolve() -- *singular* -- so with a resolver + // that reports resolved addresses as unresolved, every candidate would be redirected again onto + // the resolver's first answer: the dead address, N times over. Multi-address fallback would + // silently do nothing, and the endpoint pinned onto the channel would name an address the + // channel is not connected to -- which is what the SSL engine's peer host and + // DefaultTopologyMonitor#savePort are then derived from. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the reachable address was actually reached. This holds whichever candidate rotate() + // starts from, and it is precisely what fails when the clone re-resolves: the dead address is + // the resolver's first answer, so both attempts would land there and the connect would fail. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the hostname is expanded once, by ChannelFactory; the candidates are not re-resolved") + .containsExactly(HOSTNAME); + } + + @Test + public void should_resolve_and_connect_on_the_same_event_loop() throws InterruptedException { + // Resolution and channel registration must share the loop picked once per connect. Taking one + // loop for resolution and letting the registration pick another would advance the group's + // round-robin chooser twice per connect, parking every channel on half the loops with the + // default power-of-two chooser. The base's single-thread group would make this assertion + // vacuous, so use two loops -- on which the split behavior was deterministic. + DefaultEventLoopGroup twoLoops = new DefaultEventLoopGroup(2); + try { + when(nettyOptions.ioEventLoopGroup()).thenReturn(twoLoops); + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess( + channel -> + assertThat((Object) channel.eventLoop()) + .as("the channel must be registered on the loop resolution ran on") + .isSameAs(resolverGroup.resolverExecutor)); + } finally { + twoLoops.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS).sync(); + } + } + + @Test + public void should_fail_future_when_resolver_throws_synchronously() { + // Given – a broken custom resolver that throws instead of returning a failed future. The throw + // happens inside an event-loop task, where nothing else would ever complete the connect future: + // nothing at this stage has a timeout, so before the blanket catch in resolveCandidates() this + // hung the connect attempt (and with it control-connection init) forever. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("broken resolver"); + installResolver(new ThrowingAddressResolverGroup(failure)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** A resolver whose every method throws, standing in for a broken third-party implementation. */ + private static class ThrowingAddressResolverGroup extends AddressResolverGroup { + + private final RuntimeException failure; + + ThrowingAddressResolverGroup(RuntimeException failure) { + this.failure = failure; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + throw failure; + } + + @Override + public boolean isResolved(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve( + SocketAddress address, Promise promise) { + throw failure; + } + + @Override + public Future> resolveAll(SocketAddress address) { + throw failure; + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + throw failure; + } + + @Override + public void close() { + // nothing to do + } + }; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java new file mode 100644 index 00000000000..988b1ebe4f0 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies that a successfully connected {@link DriverChannel} carries an endpoint bound to the + * address the connection actually used, not the multi-address original. + * + *

Without this, a hostname shared by several nodes would let a later reconnect land on a + * different node while still being treated as the original {@code host_id}: {@code + * DefaultTopologyMonitor#buildNodeEndPoint} stores the channel's endpoint for the control node, and + * {@code ControlConnection} skips identity re-resolution for nodes that already have a host id. See + * {@link PinnableEndPoint}. + */ +public class ChannelFactoryPinnedEndPointTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryPinnedEndPointTest.class.getSimpleName() + "-unreachable"); + + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_pin_channel_endpoint_to_the_address_that_connected() { + // Given – an endpoint reporting a name, which the resolver expands to a dead address and the + // running local server. Whichever of the two the connection ends up on, the channel must carry + // that one. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); + ChannelFactory factory = newChannelFactory(); + TestPinnableEndPoint endPoint = new TestPinnableEndPoint(HOSTNAME); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture) + .isSuccess( + channel -> { + EndPoint channelEndPoint = channel.getEndPoint(); + // The channel resolves to the address it is actually connected to -- the name it was + // built from is gone from resolve(), which is what SSL engines and authenticators + // need. + assertThat(channelEndPoint.resolve()).isEqualTo(reachable); + // ...while still denoting the same node, so node lookups and metric names are stable. + assertThat(channelEndPoint).isEqualTo(endPoint); + assertThat(channelEndPoint.asMetricPrefix()).isEqualTo(endPoint.asMetricPrefix()); + }); + } + + @Test + public void should_leave_non_pinnable_endpoints_untouched() { + // A third-party EndPoint that does not implement PinnableEndPoint must reach the channel + // exactly + // as it was given, so existing implementations keep working. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel.getEndPoint()).isSameAs(SERVER_ADDRESS)); + } + + @Test + public void should_not_pin_to_an_unresolved_address() { + // Given – Bootstrap.disableResolver(), one of the paths where resolveCandidates hands the + // endpoint's own address straight back. For an endpoint that reports a name, the candidate is + // therefore still that name. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + ((Bootstrap) invocation.getArgument(0)).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + List pinnedTo = new ArrayList<>(); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + pinnedTo.add(resolvedAddress); + return super.pinTo(resolvedAddress); + } + }; + + // When – the connect itself cannot succeed against a name nothing resolves; what matters is + // what happened before it was attempted. + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – pinTo() is documented to take an already-resolved address. Pinning a name would freeze + // the endpoint on something that still re-expands on every connect, and for endpoints that stop + // consulting their own source once pinned, silence that source for good. + assertThatStage(channelFuture).isFailed(); + assertThat(pinnedTo).as("pinTo() must not be handed an unresolved address").isEmpty(); + } + + @Test + public void should_fail_future_when_pin_to_throws() { + // pinTo() runs in the continuation after resolution, whose exceptions CompletionStage + // swallows; a throwing implementation must fail the connect future rather than hang it. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Collections.singletonList(reachable))); + ChannelFactory factory = newChannelFactory(); + RuntimeException failure = new IllegalStateException("pinTo blew up"); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + throw failure; + } + }; + + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** + * A {@link PinnableEndPoint} that can hold a pin to any {@link SocketAddress}, including the + * local-transport addresses these tests connect over (which {@code DefaultEndPoint} cannot). + * Identity is the unpinned address, so a pinned copy stays equal to the original — the contract + * {@link PinnableEndPoint} requires. + */ + private static class TestPinnableEndPoint implements PinnableEndPoint { + + private final SocketAddress address; + private final SocketAddress pinnedAddress; + + TestPinnableEndPoint(SocketAddress address) { + this(address, null); + } + + private TestPinnableEndPoint(SocketAddress address, SocketAddress pinnedAddress) { + this.address = address; + this.pinnedAddress = pinnedAddress; + } + + @NonNull + @Override + public SocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : address; + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + return new TestPinnableEndPoint(address, resolvedAddress); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + + @Override + public boolean equals(Object other) { + return (other instanceof TestPinnableEndPoint) + && address.equals(((TestPinnableEndPoint) other).address); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index fceb8777904..658dbd8b15d 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -33,12 +34,21 @@ import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletionStage; import org.junit.Test; public class ChannelFactoryProtocolNegotiationTest extends ChannelFactoryTestBase { + /** A local address no server is bound to: connecting to it fails immediately. */ + private static final SocketAddress UNREACHABLE = + new LocalAddress( + ChannelFactoryProtocolNegotiationTest.class.getSimpleName() + "-unreachable"); + @Test public void should_succeed_if_version_specified_and_supported_by_server() { // Given @@ -280,6 +290,213 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) }); } + @Test + public void should_not_try_next_address_of_identified_node_when_negotiation_exhausts_versions() { + // Given – an *identified* node (its host id is known, so every address its name expands to is + // that same node) whose name expands to two candidates: the same live server twice, so + // whichever the rotation picks first is irrelevant. The server rejects every protocol version. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + exhaustNegotiationLadder(); + + // Then – the second candidate must not be attempted: for a node we have already identified, a + // protocol-version rejection is a property of the node, not of the address, so replaying the + // negotiation ladder against the remaining IPs would buy nothing. Checked before the future + // assertion so that on regression the stray frame is drained; leaving it unread would block the + // server's exchanger and hang the whole suite in tearDown() instead of failing this test. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after negotiation exhaustion") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + + @Test + public void + should_try_next_address_of_unidentified_endpoint_when_negotiation_exhausts_versions() { + // Given – the same setup, but for an endpoint the driver has not identified yet: a contact + // point, before host ids have been read. Its name may well expand to addresses of *different* + // nodes, so a version rejection by the first says nothing about the second. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + false); + + // The first candidate exhausts the ladder... + exhaustNegotiationLadder(); + // ...and the second is tried all the same, replaying the ladder from the top. Before this, + // resolve-contact-points=true made each address a separate node and ControlConnection advanced + // to the next one on exactly this error; collapsing a name into one node must not lose that. + exhaustNegotiationLadder(); + + // Then + assertThat(tryReadOutboundFrame(200)) + .as("the name expands to two addresses, so there is no third attempt") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .as("each candidate negotiates on its own, so this is the last one's ladder") + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the first candidate's failure must still be reported") + .hasSize(1); + assertThat(e.getSuppressed()[0]) + .isInstanceOf(UnsupportedProtocolVersionException.class); + }); + } + + @Test + public void should_surface_the_node_wide_rejection_when_an_earlier_address_failed_on_transport() { + // Given – an identified node whose name expands to a dead address first and to the live server + // second, the server rejecting every protocol version. Every address of an identified node is + // that same node, so the rejection is a property of the node rather than of the address. + mockNegotiationLadderDownToV3(); + installResolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve()))); + ChannelFactory factory = newChannelFactory(); + // The order matters here, and only here: it is what makes the pass end with a *mixed* set of + // failures instead of the version rejection on its own. + factory.random = new KeepResolverOrder(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + true); + + // The dead address sends nothing, so the ladder below is the second candidate's. + exhaustNegotiationLadder(); + + // Then – the version rejection is what the caller must see, even though the pass also produced + // a + // transport failure that ChannelPool#handleError does not classify and would otherwise be + // preferred (see ChannelFactory#surfacedFailure: a fatal failure needs every address to agree). + // A node-wide failure is the exception to that rule: demoting it would turn the forced-down + // node + // that a single-address connect has always produced into a plain reconnect. + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the earlier address's transport failure should still be attached") + .hasSize(1); + }); + } + + /** Negotiation starts at V4 and has exactly one downgrade available, to V3. */ + private void mockNegotiationLadderDownToV3() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) + .thenReturn(Optional.of(DefaultProtocolVersion.V3)); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + } + + /** + * Plays the server side of a full negotiation ladder against one candidate address: V4 rejected, + * downgrade retry with V3 rejected, i.e. no version left to try on that address. + */ + private void exhaustNegotiationLadder() { + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V3.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + } + + @Test + public void should_fail_future_when_downgrade_lookup_throws_in_connect_listener() { + // Given – a version registry that throws when the factory looks up the downgrade. The lookup + // runs inside the Netty connect listener, which swallows throwables: without the blanket catch + // in connectToAddress() the connect future would never complete and the attempt would hang. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("registry broken"); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)).thenThrow(failure); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + // Server does not support v4, which is what sends the factory to the downgrade lookup + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + /** * Depending on the Cassandra version, an "unsupported protocol" response can use different error * codes, so we test all of them. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index ed6668a6c83..c1904c8f914 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -20,6 +20,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.ProtocolVersion; @@ -42,6 +44,7 @@ import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -53,9 +56,12 @@ import io.netty.channel.DefaultEventLoopGroup; import io.netty.channel.local.LocalChannel; import io.netty.channel.local.LocalServerChannel; +import io.netty.resolver.AddressResolverGroup; +import java.net.SocketAddress; import java.time.Duration; import java.util.Collections; import java.util.Optional; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Exchanger; import java.util.concurrent.TimeUnit; @@ -123,6 +129,9 @@ public void setup() throws InterruptedException { when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_SET_KEYSPACE_TIMEOUT)) .thenReturn(Duration.ofMillis(TIMEOUT_MILLIS)); when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_REQUESTS)).thenReturn(1); + // The reference.conf default; individual tests may lower it to exercise the cap. + when(defaultProfile.getInt(DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES)) + .thenReturn(5); when(defaultProfile.getDuration(DefaultDriverOption.HEARTBEAT_INTERVAL)) .thenReturn(Duration.ofSeconds(30)); when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT)) @@ -188,6 +197,55 @@ protected Frame readOutboundFrame() { return null; // never reached } + /** + * Like {@link #readOutboundFrame()}, but returns {@code null} instead of failing the test when no + * frame arrives within {@code timeoutMillis}. + * + *

Use this to assert that the client did not send another request. Unlike asserting via + * a failing read, it also drains a frame that does arrive: the server-side exchange in {@link + * ServerInitializer} has no timeout, so a stray unread frame would block the server event loop + * and hang the whole suite in {@link #tearDown()} instead of failing just the test. + */ + protected Frame tryReadOutboundFrame(long timeoutMillis) { + try { + return requestFrameExchanger.exchange(null, timeoutMillis, MILLISECONDS); + } catch (InterruptedException e) { + fail("unexpected interruption while waiting for outbound frame", e); + return null; // never reached + } catch (TimeoutException e) { + return null; + } + } + + /** + * A {@link Random} that turns {@link java.util.Collections#shuffle} into a no-op, so a test whose + * scenario depends on which address is tried first can rely on the order the resolver returned: + * shuffle swaps element {@code i-1} with {@code nextInt(i)}, and returning {@code i-1} swaps + * every element with itself. + * + *

Assign it to {@link ChannelFactory#random}, the injection point that exists for this. + */ + static final class KeepResolverOrder extends Random { + private static final long serialVersionUID = 1L; + + @Override + public int nextInt(int bound) { + return bound - 1; + } + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + protected void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + } + protected void writeInboundFrame(Frame requestFrame, Message response) { writeInboundFrame(requestFrame, response, requestFrame.protocolVersion); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java index 682caac198d..322af30dc84 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandlerTest.java @@ -36,7 +36,6 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; -import com.datastax.oss.driver.api.core.connection.ConnectionInitException; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.DefaultProtocolVersionRegistry; import com.datastax.oss.driver.internal.core.ProtocolVersionRegistry; @@ -50,11 +49,9 @@ import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; -import com.datastax.oss.protocol.internal.ProtocolConstants.ErrorCode; import com.datastax.oss.protocol.internal.request.AuthResponse; import com.datastax.oss.protocol.internal.request.Options; import com.datastax.oss.protocol.internal.request.Query; -import com.datastax.oss.protocol.internal.request.Register; import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.AuthChallenge; import com.datastax.oss.protocol.internal.response.AuthSuccess; @@ -159,6 +156,52 @@ public void should_initialize() { assertThat(connectFuture).isSuccess(); } + /** + * Drives initialization as far as the cluster name check, which is where node identification + * branches off. + * + * @return the connect future, so that the caller can assert on its outcome. + */ + private ChannelFuture initUpToClusterName(DriverChannelOptions options) { + channel + .pipeline() + .addLast( + ChannelFactory.INIT_HANDLER_NAME, + new ProtocolInitHandler( + internalDriverContext, + DefaultProtocolVersion.V4, + null, + END_POINT, + options, + heartbeatHandler, + false)); + + ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Startup.class); + writeInboundFrame(buildInboundFrame(requestFrame, new Ready())); + + requestFrame = readOutboundFrame(); + assertThat(((Query) requestFrame.message).query) + .isEqualTo("SELECT cluster_name FROM system.local WHERE key='local'"); + writeInboundFrame(requestFrame, TestResponses.clusterNameResponse("someClusterName")); + + return connectFuture; + } + + @Test + public void should_complete_init_after_cluster_name_check() { + // Given — no keyspace to set. The node-identity read and the REGISTER request both happen + // after initialization (through the connect hook and ChannelFactory respectively), so init + // itself ends at the cluster-name check: byte for byte the pre-multi-address exchange. + ChannelFuture connectFuture = initUpToClusterName(DriverChannelOptions.DEFAULT); + + // Then + assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); + } + // Mirrors the real reporter, which only ever sees the control connection. private void stubConfigReporter() { when(internalDriverContext.getDriverConfigReporter()) @@ -601,7 +644,10 @@ public void should_initialize_with_keyspace() { } @Test - public void should_initialize_with_events() { + public void should_not_send_register_during_init_even_when_events_are_requested() { + // REGISTER moved out of initialization: ChannelFactory sends it after the connect hook has + // accepted the channel, so a candidate about to be rejected never registers for events. Init + // itself must therefore end without a Register frame even when the options ask for events. List eventTypes = ImmutableList.of("foo", "bar"); EventCallback eventCallback = mock(EventCallback.class); DriverChannelOptions driverChannelOptions = @@ -624,12 +670,8 @@ public void should_initialize_with_events() { writeInboundFrame(readOutboundFrame(), new Ready()); writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - Frame requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -664,12 +706,10 @@ public void should_initialize_with_keyspace_and_events() { assertThat(((Query) requestFrame.message).query).isEqualTo("USE \"ks\""); writeInboundFrame(requestFrame, new SetKeyspace("ks")); - requestFrame = readOutboundFrame(); - assertThat(requestFrame.message).isInstanceOf(Register.class); - assertThat(((Register) requestFrame.message).eventTypes).containsExactly("foo", "bar"); - writeInboundFrame(requestFrame, new Ready()); - + // No Register frame: setting the keyspace is now the last init step, events are registered by + // ChannelFactory after the connect hook. assertThat(connectFuture).isSuccess(); + assertNoOutboundFrame(); } @Test @@ -745,56 +785,4 @@ public void should_fail_pending_requests_only_once_if_init_fails() { logger.detachAppender(appender); logger.setLevel(levelBefore); } - - @Test - public void should_fail_connection_when_client_routes_change_rejected() { - List eventTypes = - ImmutableList.of( - ProtocolConstants.EventType.SCHEMA_CHANGE, - ProtocolConstants.EventType.STATUS_CHANGE, - ProtocolConstants.EventType.TOPOLOGY_CHANGE, - ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - EventCallback eventCallback = mock(EventCallback.class); - DriverChannelOptions driverChannelOptions = - DriverChannelOptions.builder().withEvents(eventTypes, eventCallback).build(); - channel - .pipeline() - .addLast( - ChannelFactory.INIT_HANDLER_NAME, - new ProtocolInitHandler( - internalDriverContext, - DefaultProtocolVersion.V4, - null, - END_POINT, - driverChannelOptions, - heartbeatHandler, - false)); - - ChannelFuture connectFuture = channel.connect(new InetSocketAddress("localhost", 9042)); - - // STARTUP - writeInboundFrame(readOutboundFrame(), new Ready()); - // Cluster name check - writeInboundFrame(readOutboundFrame(), TestResponses.clusterNameResponse("someClusterName")); - - // REGISTER attempt includes CLIENT_ROUTES_CHANGE - Frame registerFrame = readOutboundFrame(); - assertThat(registerFrame.message).isInstanceOf(Register.class); - Register firstRegister = (Register) registerFrame.message; - assertThat(firstRegister.eventTypes).contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); - - // Server rejects with PROTOCOL_ERROR mentioning CLIENT_ROUTES_CHANGE - writeInboundFrame( - registerFrame, - new Error( - ErrorCode.PROTOCOL_ERROR, - "Unknown event type: " + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)); - - // Connection must fail with a clear error message - assertThat(connectFuture).isFailed(); - assertThat(connectFuture.cause()) - .isInstanceOf(ConnectionInitException.class) - .hasMessageContaining("CLIENT_ROUTES_CHANGE") - .hasMessageContaining("ScyllaDB Enterprise >= 2026.1"); - } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java new file mode 100644 index 00000000000..06f5f636cba --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code + * DnsAddressResolverGroup}), installed the way a user would install one: through {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Records what it was asked to resolve, and answers with a fixed list of addresses so tests can + * assert that every one of them is tried, and in what order. + * + *

Implements {@link AddressResolver} directly rather than extending {@code + * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect over + * Netty's local transport, which is not reachable through an {@link InetSocketAddress}. + */ +class TestAddressResolverGroup extends AddressResolverGroup { + + /** Every address this group was asked to resolve, in order. */ + final List queried = new CopyOnWriteArrayList<>(); + + /** Whether a resolver was ever obtained from this group at all. */ + volatile boolean resolverRequested; + + /** The executor the last resolver was created for, i.e. the loop resolution runs on. */ + @Nullable volatile EventExecutor resolverExecutor; + + /** The addresses to answer with, or {@code null} to fail every lookup. */ + @Nullable private final List answer; + + /** + * Whether to claim that every address still needs resolving, even one that already carries an IP. + * A real resolver may do this to redirect traffic, and Netty honours it: {@code + * Bootstrap#doResolveAndConnect0} asks the resolver rather than testing the address itself. + */ + private final boolean claimNothingIsResolved; + + TestAddressResolverGroup(@Nullable List answer) { + this(answer, false); + } + + TestAddressResolverGroup(@Nullable List answer, boolean claimNothingIsResolved) { + this.answer = answer; + this.claimNothingIsResolved = claimNothingIsResolved; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + resolverRequested = true; + resolverExecutor = executor; + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + return true; + } + + @Override + public boolean isResolved(SocketAddress address) { + if (claimNothingIsResolved) { + return false; + } + // Only hostnames need resolving; anything else (including the local-transport addresses we + // hand back) is already usable. + return !(address instanceof InetSocketAddress) + || !((InetSocketAddress) address).isUnresolved(); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve(SocketAddress address, Promise promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer.get(0)); + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer); + } + + @Override + public void close() { + // nothing to do + } + }; + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java index d51ee445da5..2189d9a7847 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/control/ControlConnectionTest.java @@ -21,30 +21,45 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.channel.ChannelEvent; import com.datastax.oss.driver.internal.core.channel.DriverChannel; +import com.datastax.oss.driver.internal.core.channel.DriverChannelOptions; import com.datastax.oss.driver.internal.core.channel.MockChannelFactoryHelper; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.DefaultNodeInfo; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.NodeInfo; import com.datastax.oss.driver.internal.core.metadata.NodeStateEvent; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; import com.datastax.oss.driver.internal.core.metadata.TestNodeFactory; import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import java.net.ConnectException; +import java.net.InetSocketAddress; import java.time.Duration; +import java.util.AbstractMap.SimpleEntry; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -61,6 +76,115 @@ public void should_close_successfully_if_it_was_never_init() { assertThatStage(closeFuture).isSuccess(); } + @Test + public void should_arm_connect_hook_for_a_node_whose_host_id_is_unknown() { + // Given — a contact point: DefaultNode.newContactPoint leaves hostId null, and that is the one + // case with something to learn. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then — the identity read happens through the connect hook, inside the factory's candidate + // loop, where an address that cannot identify itself is rejected while the hostname's other + // addresses are still on hand. + DriverChannelOptions options = connectOptions(contactPoint); + assertThat(options.connectHook).isNotNull(); + assertThat(options.connectHookTimeout).isNotNull(); + } + + @Test + public void should_reject_candidate_when_hook_reads_no_host_id() { + // Given — the hook armed for a contact point, and a topology monitor that answers the identity + // read with a NodeInfo carrying no host id (a bootstrapping node). + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + NodeInfo anonymousInfo = mock(NodeInfo.class); + when(anonymousInfo.getHostId()).thenReturn(null); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(anonymousInfo)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the factory invokes the hook against the candidate channel. + CompletionStage vetted = connectOptions(contactPoint).connectHook.onConnect(channel); + + // Then — the candidate is rejected, so the factory can move on to the next address; accepting + // it would fail later in registerNode and cost the whole plan entry. + assertThatStage(vetted).isFailed(error -> assertThat(error).hasMessageContaining("host id")); + } + + @Test + public void should_not_arm_connect_hook_for_a_node_that_is_already_identified() { + // Given — node1 has a host id, so there is nothing to learn and nothing to vet: its + // connections keep the exact exchange they had before. + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(node1, channel1).build(); + + // When + controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + + // Then + assertThat(connectOptions(node1).connectHook).isNull(); + } + + @Test + public void should_use_node_info_captured_by_hook_without_reading_again() { + // Given — a contact point whose connect hook has run against the winning channel, capturing + // the identity it read. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(3, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(3); + CompletableFuture channelFuture = new CompletableFuture<>(); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .pending(contactPoint, channelFuture) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + EndPoint channelEndPoint = channel.getEndPoint(); + NodeInfo nodeInfo = mock(NodeInfo.class); + when(nodeInfo.getHostId()).thenReturn(UUID.randomUUID()); + when(nodeInfo.getEndPoint()).thenReturn(channelEndPoint); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(nodeInfo)); + DefaultNode registered = TestNodeFactory.newNode(3, context); + when(metadataManager.registerNode(nodeInfo)) + .thenReturn(CompletableFuture.completedFuture(registered)); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When — the hook runs once (as the factory would against the accepted candidate), then the + // connect completes with that same channel. + assertThatStage(connectOptions(contactPoint).connectHook.onConnect(channel)).isSuccess(); + channelFuture.complete(channel); + + // Then — what the hook captured is what gets registered, with no second identity read: the + // monitor was asked exactly once, by the hook itself. + verify(metadataManager, VERIFY_TIMEOUT).registerNode(nodeInfo); + verify(topologyMonitor, times(1)).getChannelNodeInfo(channel); + } + + /** The options {@code ChannelFactory} was asked to connect {@code node} with. */ + private DriverChannelOptions connectOptions(Node node) { + ArgumentCaptor captor = + ArgumentCaptor.forClass(DriverChannelOptions.class); + verify(channelFactory, VERIFY_TIMEOUT).connect(eq(node), captor.capture()); + return captor.getValue(); + } + @Test public void should_init_with_first_contact_point_if_reachable() { // Given @@ -964,4 +1088,309 @@ public void should_not_call_getChannelNodeInfo_for_metadata_node_with_hostId() { factoryHelper.verifyNoMoreCalls(); } + + @Test + public void should_report_auth_failure_when_every_contact_point_failed_on_authentication() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, authError(node2)))); + + assertThat(ControlConnection.isAuthFailure(error)).isTrue(); + } + + @Test + public void should_not_report_auth_failure_when_an_address_failed_on_something_else() { + // One entry no longer means one address: ChannelFactory expands a contact-point hostname to + // every address it resolves to and reports one failure for the name, with the other addresses' + // failures suppressed. Reading only the top-level throwable would call this an authentication + // failure and tell the operator their credentials are wrong, when the name's other records are + // simply unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of(new SimpleEntry<>(node1, withUnreachableSibling))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_not_report_auth_failure_when_a_node_failed_on_something_else() { + AllNodesFailedException error = + AllNodesFailedException.fromErrors( + ImmutableList.of( + new SimpleEntry<>(node1, authError(node1)), + new SimpleEntry<>(node2, new ConnectException("connection refused")))); + + assertThat(ControlConnection.isAuthFailure(error)).isFalse(); + } + + @Test + public void should_try_next_node_if_the_node_behind_a_contact_point_is_excluded() { + // Given — init on node1, then a reconnection through a contact point, the way the fallback + // plan appends the original contact points to every reconnection round. + when(reconnectionSchedule.nextDelay()).thenReturn(Duration.ofNanos(1)); + DriverChannel channel1 = newMockDriverChannel(1); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel3 = newMockDriverChannel(3); + + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(node1, channel1) + .success(contactPoint, channel2) + .success(node1, channel3) + .build(); + + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(node1); + assertThatStage(initFuture) + .isSuccess(v -> assertThat(controlConnection.channel()).isEqualTo(channel1)); + verify(eventBus, VERIFY_TIMEOUT).fire(ChannelEvent.channelOpened(node1)); + + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel2.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenAnswer( + invocation -> { + registeredNodes.put(resolvedNode.getHostId(), resolvedNode); + return CompletableFuture.completedFuture(resolvedNode); + }); + + // The backend that contact point reaches is one the policy has excluded. The event names the + // metadata node, because that is the only instance events ever name: the contact point the + // plan offered is an ephemeral object that is never their subject, so the guards that run + // before the handshake have nothing to match it against. + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When + mockQueryPlan(contactPoint, node1); + channel1.close(); + + // Then — the channel is abandoned once the handshake says which node is behind it, and the + // next node is tried. Parking here instead would be permanent: an unchanged distance fires no + // further event, so nothing downstream would ever dislodge it. + verify(reconnectionSchedule, VERIFY_TIMEOUT).nextDelay(); + factoryHelper.waitForCall(contactPoint); + verify(channel2, VERIFY_TIMEOUT).forceClose(); + verify(eventBus, never()).fire(ChannelEvent.channelOpened(resolvedNode)); + factoryHelper.waitForCall(node1); + } + + @Test + public void should_not_report_auth_failure_for_a_throwable_with_no_per_node_breakdown() { + // Only an AllNodesFailedException carries the per-node errors this question is answered from. + // Anything else has nothing to say about the credentials, and answering yes would send an + // operator whose connection was refused off to check their authentication configuration. + assertThat(ControlConnection.isAuthFailure(new ConnectException("connection refused"))) + .isFalse(); + } + + @Test + public void should_treat_a_lone_authentication_failure_as_auth_only() { + assertThat(ControlConnection.isAuthOnly(authError(node1))).isTrue(); + } + + @Test + public void + should_not_treat_an_authentication_failure_with_other_suppressed_causes_as_auth_only() { + // What the per-node log line is decided on. ChannelFactory#surfacedFailure deliberately + // promotes + // an authentication failure over transport ones, so the top-level throwable of a hostname whose + // records failed [refused, refused, auth] is the auth error -- and reporting that as "an + // authentication error" would say nothing about two thirds of the deployment being unreachable. + Throwable withUnreachableSibling = authError(node1); + withUnreachableSibling.addSuppressed(new ConnectException("connection refused")); + + assertThat(ControlConnection.isAuthOnly(withUnreachableSibling)).isFalse(); + } + + @Test + public void should_not_treat_a_non_authentication_failure_as_auth_only() { + assertThat(ControlConnection.isAuthOnly(new ConnectException("connection refused"))).isFalse(); + } + + @Test + public void should_never_pair_captured_node_info_with_a_channel_it_did_not_come_from() + throws Exception { + // The candidate loop can leave a stranded hook behind: an attempt abandoned on the hook timeout + // whose system.local response arrives anyway, and writes to the holder after the accepted + // candidate has. Held in two separate volatile fields, that late write lands between the + // accepted candidate's two writes, and a reader in that window takes the rejected candidate's + // node info as the accepted channel's -- registering the wrong host id and endpoint for the + // node + // the control connection is actually talking to. Published as one reference, the worst a late + // write can do is make the read miss and fall back to querying the open channel again. + ControlConnection.NodeInfoHolder holder = new ControlConnection.NodeInfoHolder(); + DriverChannel accepted = newMockDriverChannel(1); + DriverChannel abandoned = newMockDriverChannel(2); + NodeInfo acceptedInfo = mock(NodeInfo.class); + NodeInfo abandonedInfo = mock(NodeInfo.class); + AtomicReference mismatch = new AtomicReference<>(); + AtomicBoolean stop = new AtomicBoolean(); + + Thread writer = + new Thread( + () -> { + while (!stop.get()) { + holder.set(abandoned, abandonedInfo); + holder.set(accepted, acceptedInfo); + } + }); + writer.start(); + try { + for (int i = 0; i < 200_000 && mismatch.get() == null; i++) { + NodeInfo forAccepted = holder.getFor(accepted); + if (forAccepted != null && forAccepted != acceptedInfo) { + mismatch.set(forAccepted); + } + NodeInfo forAbandoned = holder.getFor(abandoned); + if (forAbandoned != null && forAbandoned != abandonedInfo) { + mismatch.set(forAbandoned); + } + } + } finally { + stop.set(true); + writer.join(); + } + + assertThat(mismatch.get()).isNull(); + } + + @Test + public void should_report_why_it_gave_up_when_the_node_behind_a_contact_point_is_excluded() { + // Given -- a plan with nothing but a contact point whose backend turns out to be excluded. The + // event names the metadata node, because that is the only instance events ever name. + DefaultNode contactPoint = TestNodeFactory.newContactPoint(2, context); + mockQueryPlan(contactPoint); + DriverChannel channel = newMockDriverChannel(2); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory).success(contactPoint, channel).build(); + UUID resolvedHostId = UUID.randomUUID(); + DefaultNode resolvedNode = TestNodeFactory.newNode(2, resolvedHostId, context); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + // Built outside the when(): calling a mock inside an unfinished stubbing confuses Mockito. + NodeInfo resolvedInfo = + DefaultNodeInfo.builder() + .withEndPoint(channel.getEndPoint()) + .withHostId(resolvedHostId) + .build(); + when(topologyMonitor.getChannelNodeInfo(channel)) + .thenReturn(CompletableFuture.completedFuture(resolvedInfo)); + when(metadataManager.registerNode(any())) + .thenReturn(CompletableFuture.completedFuture(resolvedNode)); + eventBus.fire(new DistanceEvent(NodeDistance.IGNORED, resolvedNode)); + + // When -- init, with no other node to fall back to. + CompletionStage initFuture = controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // Then -- the exclusion is recorded as this node's error, so the operator is told why rather + // than getting a bare NoNodeAvailableException with no cause at all. + assertThatStage(initFuture) + .isFailed( + error -> { + assertThat(error).isInstanceOf(AllNodesFailedException.class); + assertThat(((AllNodesFailedException) error).getAllErrors()) + .hasEntrySatisfying( + resolvedNode, + errors -> assertThat(errors.get(0)).hasMessageContaining("ignored")); + }); + } + + @Test + public void should_reconnect_when_the_node_a_pending_channel_reached_becomes_excluded() { + // Given -- a contact point that is a hostname, so the channel's endpoint (bound by + // ChannelFactory to the address it actually reached) is the only thing that says which node is + // at the other end while the identity read is still in flight. + DefaultNode contactPoint = + DefaultNode.newContactPoint( + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.invalid", 9042)), + context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- a metadata node at the address that channel is connected to is forced down, while the + // identity read has not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, NodeState.FORCED_DOWN, TestNodeFactory.newNode(2, context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. It + // cannot compare host ids yet, and comparing the two endpoints with equals() would resolve the + // contact-point hostname inline -- a blocking DNS lookup on the admin thread, answered by + // whichever address the name lists first, which need not even be this node. + factoryHelper.waitForCall(node1); + } + + @Test + public void should_reconnect_when_a_pending_proxy_channel_reached_an_excluded_node() { + // Given -- a Cloud-style deployment. The contact point and the metadata node are SniEndPoints + // over the same proxy and server name, and the channel carries the copy ChannelFactory pinned + // to + // the proxy IP it actually reached. resolve() therefore answers an unresolved proxy name on the + // node side and a concrete IP on the channel side, and an InetSocketAddress carrying an + // InetAddress never equals one that does not -- so comparing resolve() results here would say + // "not the control node" for every Cloud, client-route and unresolved-translator deployment. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.invalid", 9142); + String serverName = "1e9a4d0c-0000-0000-0000-00000000002a"; + SniEndPoint proxyEndPoint = new SniEndPoint(proxy, serverName); + DefaultNode contactPoint = DefaultNode.newContactPoint(proxyEndPoint, context); + mockQueryPlan(contactPoint); + DriverChannel channel2 = newMockDriverChannel(2); + when(channel2.getEndPoint()) + .thenReturn(proxyEndPoint.pinTo(new InetSocketAddress("127.0.0.2", 9142))); + DriverChannel channel1 = newMockDriverChannel(1); + MockChannelFactoryHelper factoryHelper = + MockChannelFactoryHelper.builder(channelFactory) + .success(contactPoint, channel2) + .success(node1, channel1) + .build(); + TopologyMonitor topologyMonitor = context.getTopologyMonitor(); + when(topologyMonitor.getChannelNodeInfo(channel2)).thenReturn(new CompletableFuture<>()); + + controlConnection.init(false, false, false); + factoryHelper.waitForCall(contactPoint); + + // When -- the metadata node behind that same proxy is forced down, while the identity read has + // not come back yet. + mockQueryPlan(node1); + eventBus.fire( + NodeStateEvent.changed( + NodeState.UP, + NodeState.FORCED_DOWN, + new DefaultNode(new SniEndPoint(proxy, serverName), context))); + + // Then -- the control connection recognizes that as the node it is sitting on and moves. The + // proxy endpoint's identity does not key on the address at all (proxy + server name), so + // equals() answers this without resolving anything. + factoryHelper.waitForCall(node1); + } + + private static Throwable authError(Node node) { + return new AuthenticationException(node.getEndPoint(), "mock authentication failure"); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java index b5e843d77d2..9c36cbebfee 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -29,9 +30,12 @@ import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metadata.NodeState; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.InetSocketAddress; import java.util.UUID; import org.junit.Test; import org.junit.runner.RunWith; @@ -210,6 +214,38 @@ public void should_warn_if_configured_dc_matches_no_node() { .isTrue(); } + @Test + public void should_not_warn_about_dc_mismatch_when_the_only_real_node_matches_configured_dc() { + // Given — CUSTOMER-588. A contact point given as a hostname is represented, before the control + // connection resolves it, by an ephemeral placeholder Node (built by + // MetadataManager#addContactPoints via DefaultNode#newContactPoint) whose datacenter is always + // null: it is never populated, because real topology is attached to a *different* Node object + // matched by hostId (see MetadataManager#registerNode). + // + // The removed OptionalLocalDcHelper#checkLocalDatacenterCompatibility compared the configured + // local DC against *those* placeholders, so it warned unconditionally whenever a local DC was + // configured, no matter where the contact points actually were. Here the only node carrying + // real, resolved metadata (node1) genuinely is in the configured local DC ("dc1", per base + // setup). + DefaultNode ephemeralContactPointNode = + DefaultNode.newContactPoint( + new DefaultEndPoint(new InetSocketAddress("127.0.0.9", 9042)), context); + when(metadataManager.getContactPoints()).thenReturn(ImmutableSet.of(ephemeralContactPointNode)); + DefaultLoadBalancingPolicy policy = createPolicy(); + + // When + policy.init(ImmutableMap.of(UUID.randomUUID(), node1), distanceReporter); + + // Then — no WARN at all. The retained check inspects the resolved node map, where node1 + // matches. + // Asserting that nothing is warned, rather than that one particular message is absent, also + // catches a regression that brings the false positive back under different wording. + // should_warn_if_configured_dc_matches_no_node is the positive control for this same appender, + // so a silent capture failure cannot make this pass by accident. + verify(appender, never()).doAppend(argThat(event -> event.getLevel() == Level.WARN)); + assertThat(policy.getLocalDatacenter()).isEqualTo("dc1"); + } + @NonNull protected DefaultLoadBalancingPolicy createPolicy() { return new DefaultLoadBalancingPolicy(context, DriverExecutionProfile.DEFAULT_NAME); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index f31dd2861ed..cb853f2d801 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -18,11 +18,13 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import java.io.UncheckedIOException; +import io.netty.channel.local.LocalAddress; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -66,20 +68,23 @@ public void should_fallback_when_resolve_returns_null() throws UnknownHostExcept } @Test - public void should_wrap_io_exceptions_in_unchecked_io_exception() throws UnknownHostException { + public void should_return_the_route_address_unresolved() { + // The route hostname is handed over unresolved on purpose: ChannelFactory resolves it through + // Netty's AddressResolverGroup, so a custom resolver applies to client routes too and no DNS + // lookup runs on the admin event loop that connect() is called from. UUID hostId = UUID.randomUUID(); - when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); ClientRoutesEndPoint ep = new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - assertThatThrownBy(ep::resolve) - .isInstanceOf(UncheckedIOException.class) - .hasCauseInstanceOf(UnknownHostException.class); + assertThat(ep.resolve()).isSameAs(route); + assertThat(((InetSocketAddress) ep.resolve()).isUnresolved()).isTrue(); } @Test - public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownHostException { + public void should_reflect_route_changes_on_subsequent_resolve() { UUID hostId = UUID.randomUUID(); InetSocketAddress addr1 = new InetSocketAddress("127.0.0.1", 9042); InetSocketAddress addr2 = new InetSocketAddress("10.0.0.1", 9043); @@ -96,6 +101,67 @@ public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownH assertThat(ep.resolve()).isEqualTo(addr2); } + // ---- pinTo() ------------------------------------------------------------ + + @Test + public void pin_to_should_stop_consulting_the_topology_monitor() { + UUID hostId = UUID.randomUUID(); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // No lookup at all -- that is the point: DefaultTopologyMonitor#savePort and the SSL factories + // read the channel's endpoint, and must not trigger a blocking re-resolution there. + verify(topologyMonitor, never()).resolve(hostId); + // Identity is keyed off the host id, so the pinned copy is still the same node. + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((ClientRoutesEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolve() hands out the route's hostname unresolved, and ChannelFactory passes an address + // straight back when the user disabled the resolver or a custom one declines it -- so that + // hostname can come back here. Pinning it would freeze the endpoint on a name that still + // re-expands on every connect, and silence the route lookup for good, since resolve() + // short-circuits once pinned. Both siblings return `this` for the same input. + UUID hostId = UUID.randomUUID(); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThat(endPoint.pinTo(route)).isSameAs(endPoint); + // Still asking the monitor, which is what would have been lost. + assertThat(endPoint.resolve()).isEqualTo(route); + verify(topologyMonitor, atLeastOnce()).resolve(hostId); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // Mirror DefaultEndPoint: an address that cannot be held in an InetSocketAddress field (e.g. + // the local transport used by unit tests) skips pinning rather than failing the connection. + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + // ---- equals / hashCode -------------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index a1ba4617ef5..17f9e7ef136 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -28,6 +28,8 @@ import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; import com.datastax.oss.driver.internal.core.channel.DriverChannel; @@ -66,6 +68,8 @@ public class ClientRoutesTopologyMonitorTest { @Mock private ControlConnection controlConnection; @Mock private DriverConfig driverConfig; @Mock private DriverExecutionProfile defaultProfile; + @Mock private MetadataManager metadataManager; + @Mock private Metadata metadata; private TestableClientRoutesTopologyMonitor handler; @@ -194,14 +198,21 @@ public void should_throw_after_close() { } @Test - public void should_throw_for_unresolvable_hostname() { + public void should_not_look_up_the_route_hostname() { UUID hostId = UUID.randomUUID(); - // Use a hostname guaranteed not to resolve + // A hostname guaranteed not to resolve: this must still succeed, because resolve() is a pure + // in-memory cache lookup that hands the name over unresolved. ChannelFactory resolves it later + // through Netty's AddressResolverGroup, so a custom resolver applies to client routes too and + // nothing blocks the admin event loop here. handler.setRoutes( ImmutableMap.of( hostId, new ClientRouteRecord(hostId, "this.host.does.not.exist.invalid", 9042))); - assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(UnknownHostException.class); + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result.isUnresolved()).isTrue(); + assertThat(result.getHostString()).isEqualTo("this.host.does.not.exist.invalid"); + assertThat(result.getPort()).isEqualTo(9042); } @Test @@ -220,6 +231,61 @@ public void should_refresh_updates_routes() throws UnknownHostException { assertThat(handler.resolve(hostId2)).isNotNull(); } + // ---- reresolvesNodeAddresses() ------------------------------------------- + + @Test + public void should_reresolve_when_all_known_nodes_have_client_routes() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + handler.setRoutes( + ImmutableMap.of( + hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042), + hostId2, new ClientRouteRecord(hostId2, "127.0.0.2", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + + @Test + public void should_not_reresolve_when_a_known_node_has_no_client_route() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + // Only node1 has a live client route; node2 would fall back to a static endpoint. + handler.setRoutes(ImmutableMap.of(hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + + @Test + public void should_not_reresolve_when_no_nodes_are_known_yet() { + // "Every known node has a route" is vacuously true of an empty set, and answering yes there + // suppresses the contact-point reconnection fallback at the one moment it is the only way back: + // before the first node refresh, or after this monitor has removed every node. There is nothing + // for the monitor to keep fresh, so it must not claim it is keeping anything fresh. + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(Collections.emptyMap()); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + // ---- Merge behavior tests ----------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 7da8fb39415..9c22b93e1e8 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -20,6 +20,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; import org.junit.Test; @@ -57,4 +59,140 @@ public void should_reject_null_address() { .isInstanceOf(NullPointerException.class) .hasMessage("address can't be null"); } + + @Test + public void resolve_returns_already_resolved_address_as_is() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + InetSocketAddress resolved = endPoint.resolve(); + assertThat(resolved.isUnresolved()).isFalse(); + assertThat(resolved.getHostString()).isEqualTo("127.0.0.1"); + } + + @Test + public void resolve_passes_unresolved_hostname_through_without_looking_it_up() { + // This endpoint does NOT resolve hostnames itself. It hands the unresolved address to + // ChannelFactory, which expands it through Netty's AddressResolverGroup so that a custom + // resolver installed via NettyOptions#afterBootstrapInitialized still applies -- a direct + // InetAddress.getAllByName() call here would bypass it, and would block the admin event loop + // that connect() runs on. "localhost" would resolve fine, so this assertion is only meaningful + // because we check the address comes back *unresolved*. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + + InetSocketAddress resolved = endPoint.resolve(); + + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo("localhost"); + assertThat(resolved.getPort()).isEqualTo(9042); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_hostname() { + // No lookup happens, so an unresolvable name is not an error at this level: the connect attempt + // fails later with a descriptive error instead. + DefaultEndPoint endPoint = + new DefaultEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_override_resolution_but_preserve_identity() { + InetSocketAddress hostname = InetSocketAddress.createUnresolved("test.com", 9042); + DefaultEndPoint original = new DefaultEndPoint(hostname); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + // Resolution now yields exactly the pinned address... + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + // ...but the copy still denotes the same node, and metric names must not change depending on + // which IP a connection happened to land on -- including through toString(), which is what + // TaggingMetricIdGenerator tags node metrics with, and nodes do adopt pinned copies. + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + // Equality has to hold in both directions: endpoints are used as set and map keys. + assertThat(original).isEqualTo(pinned); + // The original is untouched. + assertThat(original.resolve()).isEqualTo(hostname); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + DefaultEndPoint original = + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.com", 9042)); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((DefaultEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_return_same_instance_when_address_is_already_the_endpoints_own() { + // An already-resolved endpoint expands to exactly one candidate -- itself -- so ChannelFactory + // pins it to the address it already holds. That copy would be indistinguishable from the + // original in every respect, so there is no point allocating it. Every node discovered from the + // peers rows takes this path. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + DefaultEndPoint endPoint = new DefaultEndPoint(resolved); + + assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); + assertThat(endPoint.toString()).isEqualTo(resolved.toString()); + } + + @Test + public void pin_to_should_reject_null_address() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThatThrownBy(() -> endPoint.pinTo(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("resolvedAddress can't be null"); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // ChannelFactory pins whatever address it connected to; a non-Inet one (e.g. the local + // transport + // used by unit tests) cannot be held in an InetSocketAddress field, so pinning is skipped + // rather + // than failing the connection. + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // resolveCandidates() hands the original address straight through when the user disabled + // Netty's + // resolver, when the resolver does not support the address, or when it reports it as already + // resolved. Pinning a name would freeze resolve() on something that must re-expand on every + // connect -- no address stability gained, and the endpoint's own source silenced. The same + // guard + // is in SniEndPoint and ClientRoutesEndPoint. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("db.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void should_still_compare_an_unresolved_address_against_a_resolved_one() { + // Kept working for Metadata#findNode, whose caller may hold either form -- but it costs a DNS + // lookup on the calling thread, only compares the first address the name maps to, and does not + // agree with hashCode(). equals() warns once per JVM about it; that latch is static, so whether + // this test is the one that trips it depends on test ordering and is deliberately not asserted. + // See https://github.com/scylladb/java-driver/issues/1006. + DefaultEndPoint unresolved = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + DefaultEndPoint resolved = new DefaultEndPoint(new InetSocketAddress("localhost", 9042)); + + assertThat(unresolved).isEqualTo(resolved); + assertThat(resolved).isEqualTo(unresolved); + // ... while hashing differently, which is exactly the inconsistency the issue tracks. + assertThat(unresolved.hashCode()).isNotEqualTo(resolved.hashCode()); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java index 6a53fe3e433..f844f9d2805 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java @@ -18,10 +18,31 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.codahale.metrics.MetricRegistry; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfig; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; +import com.datastax.oss.driver.api.core.metrics.NodeMetric; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.internal.core.metrics.AbstractMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.DefaultMetricIdGenerator; +import com.datastax.oss.driver.internal.core.metrics.DropwizardNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Collections; +import java.util.Set; import java.util.UUID; import org.junit.Test; @@ -55,4 +76,239 @@ public void should_have_expected_string_representation_if_hostid_is_null() { "Node(endPoint=localhost/127.0.0.1:9042, hostId=null, hashCode=%x)", node.hashCode()); assertThat(node.toString()).isEqualTo(expected); } + + @Test + public void should_adopt_a_newer_endpoint_that_only_differs_by_its_pinned_address() { + // A PinnableEndPoint copy compares equal to the original -- pinnedAddress is excluded from + // equals() by contract, so that a pinned copy still denotes the same node. setEndPoint() must + // therefore not use equals() to decide whether to adopt it: the pinned address is the one every + // subsequent connection to this node will use, so refusing the newer instance would freeze the + // node on the first address it ever connected to, even after the control connection has moved + // and told us about it. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + DefaultNode node = new DefaultNode(endPoint, context); + + EndPoint pinnedToFirst = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)); + node.setEndPoint(pinnedToFirst, context); + assertThat(node.getEndPoint()).isSameAs(pinnedToFirst); + + EndPoint pinnedToSecond = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.3", 9042)); + // Same node by equals(), different pinned address. + assertThat(pinnedToSecond).isEqualTo(pinnedToFirst); + node.setEndPoint(pinnedToSecond, context); + + assertThat(node.getEndPoint()).isSameAs(pinnedToSecond); + assertThat(node.getEndPoint().resolve()).isEqualTo(new InetSocketAddress("127.0.0.3", 9042)); + } + + @Test + public void should_keep_the_endpoint_instance_it_already_holds_when_nothing_would_differ() { + // Every full topology refresh mints a fresh endpoint over a fresh InetSocketAddress for every + // node, and for an unchanged node it denotes exactly what this one already does. Adopting it + // would discard the reverse-DNS name that an earlier TLS handshake cached on the InetAddress + // this + // node holds (DefaultSslEngineFactory calls getHostName() under the default + // allow-dns-reverse-lookup-san = true), so the next connection would repeat that blocking + // lookup + // on an I/O event loop -- once per node per refresh instead of once per node per session. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint held = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(held, context); + + EndPoint equivalent = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(equivalent).isNotSameAs(held); + node.setEndPoint(equivalent, context); + + assertThat(node.getEndPoint()).isSameAs(held); + } + + @Test + public void should_adopt_an_endpoint_of_a_different_kind_that_resolves_to_the_same_address() { + // A dynamic endpoint (ClientRoutesEndPoint) resolves through its topology monitor on every + // call, + // so it is not interchangeable with a static one that happens to point at the same address + // today. + // The early return must not keep the static one in that case. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + EndPoint stat1c = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + DefaultNode node = new DefaultNode(stat1c, context); + + EndPoint dynamic = new RenamingEndPoint("/127.0.0.1:9042"); + assertThat(dynamic.resolve()).isEqualTo(stat1c.resolve()); + assertThat(dynamic.asMetricPrefix()).isEqualTo(stat1c.asMetricPrefix()); + + node.setEndPoint(dynamic, context); + + assertThat(node.getEndPoint()).isSameAs(dynamic); + } + + @Test + public void should_not_rebuild_the_metric_updater_for_a_pin_only_change() { + // A pinned copy is identified exactly like the original -- same asMetricPrefix(), same + // toString() -- so rebuilding would clear and re-register metrics under identical names, and + // reset their values along the way. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + DefaultNode node = new DefaultNode(endPoint, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint( + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)), context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + @Test + public void should_not_rebuild_the_metric_updater_when_only_the_endpoints_string_form_differs() { + // toString() is not usable as an identity key because it is not stable across instances that + // denote the same thing: DefaultEndPoint delegates it to InetSocketAddress, which renders + // InetAddress's cached hostName field, and that field is filled in the first time anything + // calls getHostName() -- which DefaultSslEngineFactory does, on this node's own endpoint + // instance, under the default allow-dns-reverse-lookup-san = true. Keying on it would clear and + // re-register every node's metrics on every topology refresh, since each refresh decodes a + // fresh endpoint whose hostName has not been filled in yet. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint beforeLookup = new RenamingEndPoint("/127.0.0.1:9042"); + EndPoint afterLookup = new RenamingEndPoint("host.example.com/127.0.0.1:9042"); + assertThat(beforeLookup.asMetricPrefix()).isEqualTo(afterLookup.asMetricPrefix()); + assertThat(beforeLookup.toString()).isNotEqualTo(afterLookup.toString()); + + DefaultNode node = new DefaultNode(beforeLookup, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(afterLookup, context); + + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + /** An endpoint that always reports the same metric prefix but renders differently. */ + private static class RenamingEndPoint implements EndPoint { + private final String stringForm; + + RenamingEndPoint(String stringForm) { + this.stringForm = stringForm; + } + + @NonNull + @Override + public SocketAddress resolve() { + return new InetSocketAddress("127.0.0.1", 9042); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "127_0_0_1:9042"; + } + + @Override + public String toString() { + return stringForm; + } + } + + @Test + public void should_rebuild_the_metric_updater_when_an_equal_endpoint_renames_the_metrics() { + // An unresolved hostname and the address it resolves to compare *equal* (see + // DefaultEndPoint#equals) but do not produce the same metric prefix. That is exactly what + // happens when a contact-point node adopts the endpoint built from its system.local row, so + // deciding on equals() alone would leave the node's metrics registered under the hostname while + // asMetricPrefix() had moved on to the IP. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(asHostname).isEqualTo(asAddress); + assertThat(asHostname.asMetricPrefix()).isNotEqualTo(asAddress.asMetricPrefix()); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(asAddress, context); + + assertThat(node.getMetricUpdater()).isSameAs(second); + verify(first).clearMetrics(); + } + + @Test + public void should_rebuild_the_metric_updater_without_wiping_the_metrics_it_registers() { + // The two tests above use mocks, which cannot see *when* clearMetrics() runs relative to the + // endpoint swap -- and that order is what decides whether the rebuild works. Dropwizard and + // MicroProfile do not remember the ids they registered under: clearMetrics() recomputes each + // one + // from the node's endpoint as it stands at that moment. Clearing after the swap therefore + // removes exactly the series the new updater has just registered, and leaves the old ones in + // the + // registry with nothing writing to them. So this one drives a real registry. + MetricRegistry registry = new MetricRegistry(); + NodeMetric metric = DefaultNodeMetric.UNSENT_REQUESTS; + InternalDriverContext context = dropwizardContext(registry, Collections.singleton(metric)); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + String underHostname = "s.nodes." + asHostname.asMetricPrefix() + '.' + metric.getPath(); + String underAddress = "s.nodes." + asAddress.asMetricPrefix() + '.' + metric.getPath(); + assertThat(underHostname).isNotEqualTo(underAddress); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(registry.getNames()).containsExactly(underHostname); + + node.setEndPoint(asAddress, context); + + assertThat(registry.getNames()).containsExactly(underAddress); + } + + /** A context wired to a real Dropwizard registry, enough for {@link DefaultNode} to use it. */ + private static InternalDriverContext dropwizardContext( + MetricRegistry registry, Set enabledMetrics) { + InternalDriverContext context = mock(InternalDriverContext.class); + DriverConfig config = mock(DriverConfig.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + when(context.getSessionName()).thenReturn("s"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(profile.getString(DefaultDriverOption.METRICS_ID_GENERATOR_PREFIX, "")).thenReturn(""); + // Built outside the when(): the generator's constructor reads back from the context, and + // Mockito rejects a nested call on a stubbing that is still open. + DefaultMetricIdGenerator idGenerator = new DefaultMetricIdGenerator(context); + when(context.getMetricIdGenerator()).thenReturn(idGenerator); + + MetricsFactory metricsFactory = mock(MetricsFactory.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + // Built on demand rather than up front: an updater registers its metrics from its constructor, + // under the names the node's endpoint yields at that point. + when(metricsFactory.newNodeUpdater(any())) + .thenAnswer( + invocation -> + new DropwizardNodeMetricUpdater( + invocation.getArgument(0), context, enabledMetrics, registry)); + return context; + } + + /** A context whose only stubbed behaviour is the metrics factory {@code DefaultNode} asks for. */ + private static InternalDriverContext contextWith(MetricsFactory metricsFactory) { + InternalDriverContext context = mock(InternalDriverContext.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + return context; + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java index 892715a47c3..a0ef5197734 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.ssl.SslEngineFactory; import com.datastax.oss.driver.internal.core.addresstranslation.PassThroughAddressTranslator; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; @@ -443,6 +444,76 @@ public void should_fail_get_channel_node_info_if_local_result_is_empty() { }); } + @Test + public void should_query_system_local_for_channel_node_info() { + // Given — a channel to identify (the control connection's connect hook is the caller). + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getHostId()).isEqualTo(hostId)); + } + + @Test + public void should_identify_the_connected_node_by_the_address_it_reached_not_the_contact_point() { + // Given — a control channel that came up through a contact-point hostname. ChannelFactory binds + // the endpoint it hands the channel to the one address that connection reached, and by + // PinnableEndPoint's contract that copy is identified exactly like the unpinned original. + EndPoint contactPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("db.example.com", 9042)); + EndPoint pinned = + ((PinnableEndPoint) contactPoint).pinTo(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(pinned.asMetricPrefix()).isEqualTo("db_example_com:9042"); + when(channel.getEndPoint()).thenReturn(pinned); + UUID hostId = UUID.randomUUID(); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", mockResult(mockLocalRow(1, hostId)))); + + // When + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + // Then — the node is registered under its own address, not under the name it was reached + // through. Registering it under the contact point would give it a hostname identity that is not + // even exclusively its own: the reconnection fallback hands the contact points back every + // round, + // so each successive control node would take the same metric prefix, and two live nodes sharing + // one prefix means the older one's clearMetrics() deletes the newcomer's series. + assertThatStage(futureNodeInfo) + .isSuccess( + nodeInfo -> { + assertThat(nodeInfo.getEndPoint().asMetricPrefix()).isEqualTo("127_0_0_1:9042"); + assertThat(nodeInfo.getEndPoint().resolve()) + .isEqualTo(new InetSocketAddress("127.0.0.1", 9042)); + }); + } + + @Test + public void should_keep_the_channels_own_endpoint_when_it_already_names_the_connected_address() { + // The reconnection-to-a-known-node case: nothing was pinned because the endpoint already held + // the address, so the existing instance is kept -- which is what keeps the control-node check + // in + // refreshNode() an identity comparison rather than an endpoint equals() (and DefaultEndPoint's + // equals resolves names). + EndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + when(channel.getEndPoint()).thenReturn(endPoint); + topologyMonitor.stubQueries( + new StubbedQuery( + "SELECT * FROM system.local WHERE key='local'", + mockResult(mockLocalRow(1, UUID.randomUUID())))); + + CompletionStage futureNodeInfo = topologyMonitor.getChannelNodeInfo(channel); + + assertThatStage(futureNodeInfo) + .isSuccess(nodeInfo -> assertThat(nodeInfo.getEndPoint()).isSameAs(endPoint)); + } + @Test public void should_stop_executing_queries_once_closed() { // Given diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java index 89b36b9ee09..af0d331c98c 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java @@ -33,17 +33,17 @@ import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy.DistanceReporter; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; -import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.core.metadata.Metadata; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeState; import com.datastax.oss.driver.internal.core.context.EventBus; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; -import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.Map; import java.util.Objects; import java.util.Queue; @@ -79,6 +79,7 @@ public class LoadBalancingPolicyWrapperTest { private EventBus eventBus; @Mock private MetadataManager metadataManager; @Mock private Metadata metadata; + @Mock private TopologyMonitor topologyMonitor; @Mock protected MetricsFactory metricsFactory; @Captor private ArgumentCaptor> initNodesCaptor; @@ -102,13 +103,16 @@ public void setup() { when(metadata.getNodes()).thenReturn(allNodes); when(metadataManager.getContactPoints()).thenReturn(contactPoints); when(context.getMetadataManager()).thenReturn(metadataManager); + when(context.getTopologyMonitor()).thenReturn(topologyMonitor); when(context.getConfig()).thenReturn(config); when(config.getDefaultProfile()).thenReturn(defaultProfile); when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(false); - defaultPolicyQueryPlan = Lists.newLinkedList(ImmutableList.of(node3, node2, node1)); + // Use a real built-in QueryPlan (not a mutable LinkedList): its add()/addAll() throw, so the + // control-reconnection plan must compose rather than mutate it (see CompositeQueryPlan usage). + defaultPolicyQueryPlan = new SimpleQueryPlan(node3, node2, node1); when(policy1.newQueryPlan(null, null)).thenReturn(defaultPolicyQueryPlan); eventBus = spy(new EventBus("test")); @@ -130,26 +134,28 @@ public void setup() { @Test public void should_build_control_connection_query_plan_from_contact_points_before_init() { - // When + // When — before init, the control-reconnection plan is built straight from the contact points + // (bypassing the load balancing policies), so each hostname can be tried on the first connect. Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test public void should_build_query_plan_from_contact_points_before_init() { - // When + // When — before init, the query plan is built straight from the contact points (bypassing the + // load balancing policies) Queue queryPlan = wrapper.newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test @@ -204,17 +210,106 @@ public void should_fetch_control_connection_query_plan_from_policy_after_init() assertThat(queryPlan.poll()).isEqualTo(node3); assertThat(queryPlan.poll()).isEqualTo(node2); assertThat(queryPlan.poll()).isEqualTo(node1); - // Remaining nodes are contact points appended at the end. - // They are new DefaultNode instances created via newContactPoint, so compare by endpoint. - Set remainingEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - remainingEndpoints.add(n.getEndPoint()); + // Remaining nodes are the original contact points appended at the end. DefaultNode does not + // override equals, so comparing against the retained instances is an identity check. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); + } + + @Test + public void should_reuse_the_retained_contact_point_nodes_rather_than_minting_copies() { + // Given — the flag now defaults to true, so this plan is built on every control-connection + // reconnection round for every user, not only for those who opted in. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When — two rounds, as a reconnection sequence would produce. + Queue firstPlan = wrapper.newControlReconnectionQueryPlan(); + Queue secondPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — both plans hand out the very objects MetadataManager retains, not per-plan copies. + // A copy would be invisible to the metric updater registered for the real node and would fire + // its own controlConnectionFailed event, once per contact point per round, for as long as + // reconnection lasts. + for (Node node : firstPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); + for (Node node : secondPlan) { + assertThat(contactPoints).contains((DefaultNode) node); } - assertThat(remainingEndpoints).isEqualTo(contactEndpoints); + // And the retained set itself is untouched by the shuffle. + assertThat(contactPoints).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void should_not_duplicate_contact_points_before_init() { + // Given — the wrapper hasn't been init()-ed yet (state=BEFORE_INIT), so newQueryPlan() already + // builds the regular plan directly from the contact points. The reconnect-contact-points flag + // doesn't matter here: newControlReconnectionQueryPlan() short-circuits on state before even + // reading it, since appending contact points again pre-init would just duplicate every entry in + // the plan. + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the contact points are read only once (not once for the regular plan and again for a + // redundant "fallback" append), and the plan has no duplicate entries. + verify(metadataManager, times(1)).getContactPoints(); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_reconnect_contact_points_is_disabled() { + // Given — the flag defaults to false in the test setup (see @Before) + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Only the policy query plan is returned; no contact points are appended. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_topology_monitor_reresolves_addresses() { + // Given — the flag is enabled, but the topology monitor re-resolves node addresses on its own + // (e.g. a proxy-based monitor such as client routes or the cloud SNI proxy). + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Contact points must not be appended: the monitor keeps addresses fresh, and appending raw + // contact points could resurrect nodes it has authoritatively removed. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_append_contact_points_when_query_plan_empty_even_if_topology_monitor_reresolves() { + // Given — the flag is enabled and the topology monitor re-resolves node addresses on its own, + // but the live-node query plan is empty. With no node to try, reconnection can only recover + // through the contact-point fallback, so it must be appended despite the re-resolving monitor. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the retained contact-point instances are appended, in some order. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test @@ -223,24 +318,15 @@ public void should_return_contact_points_when_query_plan_empty_and_flag_enabled( when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(true); wrapper.init(); - // Make the policy return an empty query plan - when(policy1.newQueryPlan(null, null)).thenReturn(Lists.newLinkedList(ImmutableList.of())); + // Make the policy return an empty query plan (QueryPlan.EMPTY, as the real policies do) + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); // When Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); // Then - // Should get the contact points (compare by endpoint since they are new instances) - assertThat(queryPlan.size()).isEqualTo(contactPoints.size()); - Set resultEndpoints = new java.util.HashSet<>(); - for (Node n : queryPlan) { - resultEndpoints.add(n.getEndPoint()); - } - Set contactEndpoints = new java.util.HashSet<>(); - for (DefaultNode n : contactPoints) { - contactEndpoints.add(n.getEndPoint()); - } - assertThat(resultEndpoints).isEqualTo(contactEndpoints); + // Should get the retained contact-point instances themselves. + assertThat(queryPlan).containsExactlyInAnyOrderElementsOf(contactPoints); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java new file mode 100644 index 00000000000..416821798d9 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class SniEndPointTest { + + @Test + public void resolve_returns_the_proxy_address_as_is_without_looking_it_up() { + // The proxy address is a hostname (that is how CloudConfigFactory builds it) and this endpoint + // must not resolve it: ChannelFactory expands it through Netty's AddressResolverGroup, which is + // what makes a custom resolver apply to the SNI proxy too, and what keeps resolve() safe to + // call + // from an event loop -- SniSslEngineFactory#newSslEngine does exactly that. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + SniEndPoint endPoint = new SniEndPoint(proxy, "test-server-name"); + + assertThat(endPoint.resolve()).isSameAs(proxy); + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + } + + @Test + public void should_keep_a_resolved_proxy_hostname_unresolved() { + // InetSocketAddress(String, int) resolves eagerly, so a hostname passed to + // withCloudProxyAddress() arrives here already bound to one of its IPs. Storing it that way + // would freeze every Cloud connection on that IP for the life of the session: resolve() hands + // the stored address straight to the connection layer, which only expands unresolved ones. + InetSocketAddress resolvedProxy = new InetSocketAddress("localhost", 9042); + assertThat(resolvedProxy.isUnresolved()).isFalse(); + + SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("localhost"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + // Normalization is unconditional, so endpoints built from either form of the same proxy still + // denote the same node -- equals() keys on the stored address. + assertThat(endPoint) + .isEqualTo( + new SniEndPoint( + InetSocketAddress.createUnresolved("localhost", 9042), "test-server-name")); + // The metric prefix is unaffected either way: it was already built from the host string. + assertThat(endPoint.asMetricPrefix()).isEqualTo("localhost:9042_test-server-name"); + } + + @Test + public void should_store_a_proxy_given_as_an_ip_address_unresolved_too() { + // An IP literal has nothing to expand, but it is stored unresolved all the same, because that + // is what makes this endpoint's identity immune to drift: see the test below. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("127.0.0.1"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void identity_should_not_drift_when_the_source_address_grows_a_reverse_dns_name() { + // SniSslEngineFactory#newSslEngine calls getHostName() on the address resolve() hands it, under + // the default allow-dns-reverse-lookup-san = true. On a *resolved* address that caches the + // reverse-DNS name on the underlying InetAddress, and getHostString() reports it from then on. + // An endpoint keyed on such an instance would silently change its metric prefix mid-session, + // moving a node's metrics; storing the address unresolved removes the field that changes. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + InetSocketAddress storedBefore = endPoint.resolve(); + + // What the SSL engine factory does. 127.0.0.1 has a PTR record on essentially every machine, so + // this really does change the source instance's host string. + ipProxy.getHostName(); + assertThat(ipProxy.getHostString()).isNotEqualTo("127.0.0.1"); + + assertThat(endPoint.resolve()).isEqualTo(storedBefore); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + assertThat(endPoint).isEqualTo(new SniEndPoint(storedBefore, "test-server-name")); + + // And the driver no longer pollutes the address it holds in the first place: getHostName() on + // an + // unresolved address returns its stored host string without looking anything up, so the call + // above cannot happen to *this* instance. (What the SSL engine actually sees is the pinned + // copy, + // whose address is excluded from equals() and asMetricPrefix() by contract.) + assertThat(endPoint.resolve().getHostName()).isEqualTo("127.0.0.1"); + assertThat(endPoint.asMetricPrefix()).isEqualTo("127_0_0_1:9042_test-server-name"); + } + + @Test + public void resolve_does_not_throw_for_unresolvable_proxy_hostname() { + // No lookup happens here, so an unresolvable name only fails later, at connect time. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042), + "test-server-name"); + + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); + } + + @Test + public void pin_to_should_make_resolve_return_the_connected_proxy_ip_and_preserve_identity() { + // Pinning is what lets SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel + // initializer -- see the very proxy IP the channel is connected to, rather than the hostname or + // another A-record. + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolve().isUnresolved()).isFalse(); + // The original is untouched. + assertThat(original.resolve().isUnresolved()).isTrue(); + + // The pinned copy still denotes the same node, down to every string it is identified by: the + // tagging MetricIdGenerator tags node metrics with the endpoint's toString(), and nodes do + // adopt + // pinned copies. + assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_an_unresolved_address() { + // ChannelFactory hands the address straight back when the user disabled Netty's resolver or a + // custom one declines it. Pinning that would freeze resolve() on a name that must re-expand on + // every connect, silencing the proxy's A-record fallback for good. + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(InetSocketAddress.createUnresolved("proxy.example.com", 9042))) + .isSameAs(endPoint); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + SniEndPoint endPoint = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + + assertThat(endPoint.pinTo(new LocalAddress("test"))).isSameAs(endPoint); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java index ccc42a7027d..c31a77324e9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metrics/DropwizardNodeMetricUpdaterTest.java @@ -18,8 +18,11 @@ package com.datastax.oss.driver.internal.core.metrics; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,13 +36,18 @@ import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.NodeMetric; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.util.LoggerTest; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; import java.time.Duration; import java.util.Collections; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.junit.Test; import org.junit.runner.RunWith; @@ -152,6 +160,118 @@ protected void initializeHdrTimer( verify(logger.appender, timeout(500).times(0)).doAppend(logger.loggingEventCaptor.capture()); } + @Test + public void should_adopt_a_pending_expiration_from_the_updater_it_replaces() { + // given – two updaters for the same node, as DefaultNode.setEndPoint builds when an endpoint + // change renames the metrics, and an expiration already armed on the one being replaced. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout pendingTimeout = mock(Timeout.class); + Timeout adoptedTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(pendingTimeout, adoptedTimeout); + // Still pending, i.e. the countdown had not run yet: that is what makes it worth carrying over. + when(pendingTimeout.cancel()).thenReturn(true); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // when + replacement.adoptExpirationFrom(previous); + + // then – the countdown is cancelled on the object nothing refers to any more and re-armed on + // the + // one whose metrics it will clear. Without this the expiration is simply lost: it is armed and + // cancelled through node.getMetricUpdater(), so a node that is already down never produces + // another DOWN event to arm the replacement, while the orphan timer still fires clearMetrics() + // on names recomputed from whatever endpoint the node holds by then. + verify(pendingTimeout).cancel(); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + // and – there is nothing left to hand over a second time. + replacement.adoptExpirationFrom(previous); + verify(timer, times(2)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + @Test + public void should_not_re_arm_an_expiration_that_has_already_fired() { + // The timer task is clearMetrics() followed by cancelMetricsExpirationTimeout(), so an endpoint + // change landing between the two finds a non-null reference to a timeout that has already run. + // Re-arming on that would start a fresh expire-after countdown and wipe the replacement's newly + // registered metrics an hour later, even though this node's metrics were already cleared once. + Node node = mock(Node.class); + InternalDriverContext context = mock(InternalDriverContext.class); + DriverExecutionProfile profile = mock(DriverExecutionProfile.class); + DriverConfig config = mock(DriverConfig.class); + NettyOptions nettyOptions = mock(NettyOptions.class); + Timer timer = mock(Timer.class); + Timeout expiredTimeout = mock(Timeout.class); + when(context.getSessionName()).thenReturn("prefix"); + when(context.getConfig()).thenReturn(config); + when(config.getDefaultProfile()).thenReturn(profile); + when(profile.getDuration(DefaultDriverOption.METRICS_NODE_EXPIRE_AFTER)) + .thenReturn(AbstractMetricUpdater.MIN_EXPIRE_AFTER); + when(context.getNettyOptions()).thenReturn(nettyOptions); + when(nettyOptions.getTimer()).thenReturn(timer); + when(timer.newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class))) + .thenReturn(expiredTimeout); + // Timeout.cancel() returns false once the task has run (or was already cancelled). + when(expiredTimeout.cancel()).thenReturn(false); + + DropwizardNodeMetricUpdater previous = newNodeUpdater(node, context); + DropwizardNodeMetricUpdater replacement = newNodeUpdater(node, context); + previous.startMetricsExpirationTimeout(); + verify(timer).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + + replacement.adoptExpirationFrom(previous); + + verify(timer, times(1)).newTimeout(any(TimerTask.class), anyLong(), any(TimeUnit.class)); + } + + /** A node updater with metric registration stubbed out: only its expiration behavior matters. */ + private static DropwizardNodeMetricUpdater newNodeUpdater( + Node node, InternalDriverContext context) { + return new DropwizardNodeMetricUpdater( + node, + context, + Collections.singleton(DefaultNodeMetric.CQL_MESSAGES), + new MetricRegistry()) { + @Override + protected void initializeGauge( + NodeMetric metric, DriverExecutionProfile profile, Supplier supplier) { + // do nothing + } + + @Override + protected void initializeCounter(NodeMetric metric, DriverExecutionProfile profile) { + // do nothing + } + + @Override + protected void initializeHdrTimer( + NodeMetric metric, + DriverExecutionProfile profile, + DriverOption highestLatency, + DriverOption significantDigits, + DriverOption interval) { + // do nothing + } + }; + } + @DataProvider public static Object[][] acceptableEvictionTimes() { return new Object[][] { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java new file mode 100644 index 00000000000..f48d70ef838 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class AddressUtilsTest { + + @Test + public void should_recognize_a_hostname_whether_resolved_or_not() { + assertThat( + AddressUtils.carriesName(InetSocketAddress.createUnresolved("host.example.com", 9042))) + .isTrue(); + // Eagerly resolved by the constructor, but still a name. + assertThat(AddressUtils.carriesName(new InetSocketAddress("localhost", 9042))).isTrue(); + } + + @Test + public void should_not_mistake_an_ip_literal_for_a_hostname() throws Exception { + assertThat(AddressUtils.carriesName(new InetSocketAddress("127.0.0.1", 9042))).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("127.0.0.1", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("::1", 9042))).isFalse(); + // Built from raw bytes, so it carries no name at all and getHostString() falls back to the + // literal -- without triggering the reverse lookup that getHostName() would. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042))) + .isFalse(); + } + + @Test + public void should_report_an_explicitly_named_address_as_a_name() throws Exception { + // A resolver may label its results with a name of its own; that is still a name. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress( + InetAddress.getByAddress("cname.example.com", new byte[] {10, 0, 0, 1}), 9042))) + .isTrue(); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java index 9a23f368546..e658f843360 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -186,12 +186,12 @@ private void requireSystemClientRoutesTable(CqlSession admin) { } private InetSocketAddress tryResolve(ClientRoutesTopologyMonitor handler, UUID hostId) { + // resolve() is an in-memory cache lookup that hands the route hostname over unresolved -- the + // connection layer resolves it -- so the only failure left is the monitor being closed. try { return handler.resolve(hostId); } catch (IllegalStateException e) { return null; - } catch (UnknownHostException e) { - throw new RuntimeException("DNS resolution failed for host_id=" + hostId, e); } } @@ -206,7 +206,9 @@ private NodeClassification classifyNodes(CqlSession session) { NodeClassification result = new NodeClassification(); for (Node node : session.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // getHostString() rather than getAddress().getHostAddress(): a client route is handed over + // unresolved (the connection layer resolves it), so getAddress() is null for proxied nodes. + String ip = addr.getHostString(); UUID hostId = node.getHostId(); boolean connected = node.getOpenConnections() > 0; LOG.info( @@ -319,7 +321,8 @@ private Map collectHostIds(CcmBridge ccm, int nodeCount, String t .build()) { for (Node node : adminSession.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // See classifyNodes(): a client route is unresolved, so getAddress() would be null. + String ip = addr.getHostString(); Integer nodeId = ipToNodeId.get(ip); if (nodeId != null && node.getHostId() != null) { hostIds.put(nodeId, node.getHostId()); @@ -540,7 +543,10 @@ public void should_refresh_routes_after_table_update() throws Exception { () -> { InetSocketAddress resolved = handler.resolve(hostId); assertThat(resolved).isNotNull(); - assertThat(resolved.getAddress().getHostAddress()).isEqualTo(nodeAddr); + // The route is returned unresolved on purpose -- ChannelFactory resolves it through + // Netty's AddressResolverGroup -- so assert on the host string, not getAddress(). + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo(nodeAddr); assertThat(resolved.getPort()).isEqualTo(9042); }); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java index 26658bd76d1..dcede822fca 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java @@ -235,6 +235,12 @@ private CqlSession newSession(ProgrammaticDriverConfigLoaderBuilder loaderBuilde .withDuration(DefaultDriverOption.HEARTBEAT_TIMEOUT, Duration.ofMillis(500)) .withDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT, Duration.ofSeconds(2)) .withDuration(DefaultDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(1)) + // These tests exercise heartbeat behavior only. The contact-point reconnection + // fallback appends the contact points to a reconnection plan that has run out of live + // nodes, so a round tries roughly twice as many addresses -- and every one of those + // attempts sends an OPTIONS as part of protocol init. countHeartbeats() filters on + // isOptionRequest() alone, so it counts those alongside real heartbeats. Disable it. + .withBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false) .build(); return SessionUtils.newSession(SIMULACRON_RULE, loader); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index 4e9eefebf63..4f8ba6897f8 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -25,9 +25,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; @@ -37,6 +39,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.config.typesafe.DefaultProgrammaticDriverConfigLoaderBuilder; import java.net.InetSocketAddress; import java.util.Collection; @@ -44,9 +47,12 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -60,6 +66,47 @@ public class MockResolverIT { private static final int CLUSTER_WAIT_SECONDS = 20; // Maximal wait time for cluster nodes to get up + /** + * A loopback address no node is ever started on, outside the {@code 127.0.1.} prefix CCM hands + * its nodes: a connection attempt that dials it is refused immediately. + */ + private static final String DEAD_ADDRESS = "127.0.0.11"; + + private static final ch.qos.logback.classic.Logger CHANNEL_FACTORY_LOGGER = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(ChannelFactory.class); + + private ListAppender channelFactoryAppender; + private Level originalChannelFactoryLevel; + + @Before + public void startCapturingChannelFactoryLogs() { + // ChannelFactory reports a candidate it gave up on at DEBUG, which is the only externally + // visible evidence that the multi-address fallback did any work. + originalChannelFactoryLevel = CHANNEL_FACTORY_LOGGER.getLevel(); + CHANNEL_FACTORY_LOGGER.setLevel(Level.DEBUG); + channelFactoryAppender = new ListAppender<>(); + // ChannelFactory logs from every I/O thread of the session under test, and the assertions read + // the list while those threads are still winding down; ListAppender's own ArrayList is not safe + // for that (appends are unsynchronized, and iterating one would throw). + channelFactoryAppender.list = new CopyOnWriteArrayList<>(); + channelFactoryAppender.start(); + CHANNEL_FACTORY_LOGGER.addAppender(channelFactoryAppender); + } + + @After + public void stopCapturingChannelFactoryLogs() { + CHANNEL_FACTORY_LOGGER.detachAppender(channelFactoryAppender); + channelFactoryAppender.stop(); + CHANNEL_FACTORY_LOGGER.setLevel(originalChannelFactoryLevel); + } + + /** The formatted messages {@code ChannelFactory} logged during the current test. */ + private List channelFactoryLogMessages() { + return channelFactoryAppender.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .collect(Collectors.toList()); + } + private static void waitForAllNodesUp(CqlSession session, int expectedNodes) { Awaitility.await() .atMost(CLUSTER_WAIT_SECONDS, TimeUnit.SECONDS) @@ -84,7 +131,6 @@ public void should_connect_with_mocked_hostname() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -107,19 +153,82 @@ public void should_connect_with_mocked_hostname() { .filter(x -> x.toString().contains("test.cluster.fake")) .collect(Collectors.toSet()); assertThat(filteredNodes).hasSize(1); - InetSocketAddress address = - (InetSocketAddress) filteredNodes.iterator().next().getEndPoint().resolve(); - assertTrue(address.isUnresolved()); + Node node = filteredNodes.iterator().next(); + InetSocketAddress address = (InetSocketAddress) node.getEndPoint().resolve(); + // ChannelFactory pins the control connection's endpoint to the address it actually reached, + // and DefaultTopologyMonitor#buildNodeEndPoint stores that copy for the control node, so + // resolution yields that concrete IP rather than the hostname. + assertFalse(address.isUnresolved()); + assertThat(address.getAddress().getHostAddress()).isEqualTo(ccmBridge.getNodeIpAddress(1)); + // The pinned copy still denotes the same node by hostname though -- it is what the filter + // above matched on -- so metric names do not depend on which IP a connection landed on. + assertThat(node.getEndPoint().asMetricPrefix()).isEqualTo("test_cluster_fake:9042"); } } } + @Test + public void should_connect_when_first_dns_entry_is_non_responsive() { + final int numberOfNodes = 2; + DriverConfigLoader loader = + new DefaultProgrammaticDriverConfigLoaderBuilder() + .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) + .withStringList( + TypedDriverOption.CONTACT_POINTS.getRawOption(), + Collections.singletonList("test.cluster.fake:9042")) + .build(); + + CqlSessionBuilder builder = new CqlSessionBuilder().withConfigLoader(loader); + try (CcmBridge ccmBridge = + CcmBridge.builder().withNodes(numberOfNodes).withIpPrefix("127.0.1.").build()) { + MultimapHostResolverProvider.removeResolverEntries("test.cluster.fake"); + // Nothing is ever started on DEAD_ADDRESS, so it is the dead record. + MultimapHostResolverProvider.addResolverEntry("test.cluster.fake", DEAD_ADDRESS); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(1)); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(2)); + ccmBridge.create(); + ccmBridge.start(); + + // ChannelFactory shuffles a name's expanded addresses per connection attempt, so whether the + // dead record is dialed first is a coin toss (about 1 in 3 here). Every session must connect + // either way; the loop hunts for an iteration that demonstrably dialed the dead record first + // and fell through to a live one, which is the behavior this test exists to pin down. + // Twenty misses in a row would have probability (2/3)^20, about 0.03%. + boolean sawFallback = false; + for (int attempt = 0; attempt < 20 && !sawFallback; attempt++) { + try (CqlSession session = builder.build()) { + waitForAllNodesUp(session, numberOfNodes); + ResultSet rs = session.execute("select * from system.local where key='local'"); + assertThat(rs).isNotNull(); + List rows = rs.all(); + assertThat(rows).hasSize(1); + Collection nodes = session.getMetadata().getNodes().values(); + assertThat(nodes).hasSize(numberOfNodes); + } + sawFallback = + channelFactoryLogMessages().stream() + .anyMatch( + message -> + message.contains(DEAD_ADDRESS) && message.contains("trying next address")); + } + + // The connections above only survived a dead-first ordering because the candidate loop moved + // past the dead record. + assertThat(sawFallback) + .as( + "expected at least one connection attempt to dial %s first and fall through", + DEAD_ADDRESS) + .isTrue(); + } + } + @Test public void replace_cluster_test() { final int numberOfNodes = 3; DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -206,7 +315,6 @@ public void run_replace_test_20_times() { public void cannot_reconnect_with_resolved_socket() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index ae44feea3ea..c692aa63336 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -183,13 +183,19 @@ datastax-java-driver { #### DNS resolution -DNS is resolved at connection time (not at route discovery time). The driver delegates to -`InetAddress.getByName()`, which is a blocking call that uses the JVM's built-in DNS cache -(30 s default TTL in the JDK). Because this runs on Netty I/O threads, slow or unresponsive -DNS can block connection establishment and impact driver throughput. To mitigate this, configure -the JVM DNS cache TTL via the `networkaddress.cache.ttl` security property (e.g. in -`$JAVA_HOME/conf/security/java.security` or programmatically with -`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`). +DNS is resolved at connection time (not at route discovery time), and through the same mechanism as +every other address the driver connects to: the route's hostname is handed to the connection layer +unresolved, and Netty's configured `AddressResolverGroup` expands it. A custom resolver installed via +`NettyOptions.afterBootstrapInitialized()` therefore applies to client routes as well, and a hostname +that maps to several addresses has all of them tried in turn. + +With Netty's default (JDK) resolver the lookup is a blocking `InetAddress` call that uses the JVM's +built-in DNS cache (30 s default TTL in the JDK). It runs on a Netty I/O event loop — never on the +admin event loop that drives control-connection reconnects — so it delays the connection attempt +itself. It is therefore worth configuring the JVM DNS cache TTL via the `networkaddress.cache.ttl` +security property (e.g. in `$JAVA_HOME/conf/security/java.security` or programmatically with +`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`), or installing +`DnsAddressResolverGroup` for non-blocking resolution. - **Route-map refresh** — the driver re-queries `system.client_routes` and atomically swaps the in-memory route map in two situations: diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 83963f94775..511f9dda2ca 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -61,6 +61,163 @@ Binary compatibility is unaffected — an already-compiled subclass keeps workin override either method, you have to widen your override to `public` in order to recompile: Java does not allow an override to reduce visibility. +#### Contact points are expanded to all their DNS addresses at connection time + +Contact points backed by a hostname are now kept unresolved and expanded to **all** the IP +addresses the hostname maps to, at connection time. Previously only the first address returned by +DNS was tried, so a single non-responsive IP behind a multi-record hostname could fail the initial +connection (or a control-connection reconnect) even when the other addresses were healthy. No +configuration change is required to benefit from this. + +The expansion goes through Netty's configured `AddressResolverGroup`, which is the resolver an +unresolved address already reached when it was handed to `Bootstrap.connect()`. A custom resolver +installed through `NettyOptions.afterBootstrapInitialized()` therefore keeps working; note that the +driver now calls `resolveAll()` on it, where previously `Bootstrap.connect()` called `resolve()`. +Every resolver built on Netty's own base classes implements both. As before, with Netty's default +(JDK) resolver the lookup blocks the I/O event loop it runs on — install `DnsAddressResolverGroup` +for non-blocking resolution. + +`Bootstrap.disableResolver()` is still honored, in the sense that the driver resolves nothing and +passes the address through as-is. That now requires the address to be usable as-is, which is a +narrower contract than before: contact-point hostnames are always kept unresolved, and the Cloud/SNI +proxy address and client-route hostnames no longer resolve themselves either, so with resolution +disabled none of them can connect. The driver fails such an attempt with a message naming the +address and the reason, rather than the bare `UnresolvedAddressException` Netty would otherwise +raise. If you disable the resolver, supply already-resolved addresses. + +The same now applies to the two other address sources that used to perform their own JVM DNS lookups: +the Cloud/SNI proxy address and client-route hostnames are expanded by the connection layer too. A +custom Netty resolver applies to them for the first time, and a proxy hostname with several A-records +has all of them tried within a single connection attempt. + +There is **no public API change**: `EndPoint.resolve()` keeps its signature and is not deprecated. +Third-party `EndPoint` implementations keep working unchanged, with one new expectation — an +implementation should return its address as-is rather than looking names up itself, since resolution +now happens in the connection layer and `resolve()` is called from an event loop. + +**If you call `node.getEndPoint().resolve()` yourself, check how you read the host.** Because +resolution moved to the connection layer, that address is no longer always resolved: + +| Deployment | `resolve()` returns | +| --- | --- | +| Nodes discovered from `system.peers`, and the node the control connection is on | resolved, as before | +| Cloud / SNI proxy | the configured proxy hostname, **unresolved** | +| Cloud private-endpoint client routes (nodes that have a route) | the route hostname, **unresolved** | + +For the last two, `InetSocketAddress.getAddress()` now returns `null`, so a call such as +`((InetSocketAddress) node.getEndPoint().resolve()).getAddress().getHostAddress()` throws a +`NullPointerException` where it previously worked. Use `getHostString()` instead: it returns +whichever of a hostname or an IP literal the address carries, for both the resolved and the +unresolved case, and never triggers a reverse lookup. + +As part of this change: + +- `advanced.resolve-contact-points` is deprecated and now has **no effect**; setting it to `true` + logs a warning at startup. Contact points are always kept as unresolved hostnames and expanded at + connection time. An already-resolved `InetSocketAddress` passed programmatically is still used as + provided, with no further expansion. + + If you had set it to `true`, note what that setting used to buy you, since the driver no longer + offers it: each address a contact-point hostname resolved to became its own `Node`, so each had + its own entry in `Metadata.getNodes()` and its own node-level metrics, and the load balancing + policy's query plan advanced address by address. A hostname is now a single `Node`, and its + addresses are tried inside one connection attempt. Resolution + also used to happen once, at startup; it now happens per connection attempt, through Netty's + configured resolver, which is what lets the driver pick up changed DNS records — but it means the + resolver is consulted far more often than before, so the JVM DNS cache settings + (`networkaddress.cache.ttl`) matter more than they used to. +- **The node the control connection is on is identified by the address it is connected to, not by + the contact point it was reached through.** This shows up in node-level metrics: with a + hostname contact point, the node the control connection came up on used to report under a prefix + derived from that hostname (`nodes.cluster_example_com:9042.*`, and the `node` tag likewise) — + which was never really its own name, since any node can end up being the one a contact point + reaches. It now reports under its own address, like every other node + (`nodes.10_0_0_2:9042.*`). If you scraped a node series named after your contact-point hostname, + expect it to move once, to that node's address. Nothing to configure. +- `advanced.control-connection.reconnection.fallback-to-original-contact-points` now defaults to + `true` (previously `false`). This is also the driver's DNS re-resolution path: a metadata node + normally holds an address decoded from the peers table, which is already resolved and so never + picks up a DNS change, and falling back to the original contact points on control-connection + reconnect is what re-reads the records once the live-node plan is exhausted. (The exception is an + `AddressTranslator` that returns a hostname — `SubnetAddressTranslator` does, unless you set + `advanced.address-translator.resolve-addresses = true`. Such an endpoint is re-expanded on every + connection attempt like any other name, so its node is not pinned to one address in the first + place. Its addresses are tried in the resolver's own order rather than shuffled — see below — so + its channels stay on one address in practice, but if that name maps to more than one host the + driver has no way to tell.) Set it to `false` to + restore the previous behavior. Note the cost: + the contact points are appended without being compared against the live-node plan (at plan time + they are still hostnames, while the live nodes are already-resolved IPs), so when DNS has not + changed a reconnection round that exhausts the live nodes retries roughly twice as many addresses. +- **A contact point that no address can identify is now marked down before the connection attempt + completes.** Establishing which node answered — reading `host_id` from `system.local` — happens + while the driver still holds the hostname's other addresses, so a candidate address that cannot + identify itself is rejected like any other per-address failure and the next address of the same + hostname is tried. When every address has been tried and none worked, the driver fires the same + `controlConnectionFailed` event and node-down state change that a refused TCP connection + produces, where previously it moved on to the next contact point without either. Nothing to + change on your side; expect the DOWN event and its log line in a case that used to be silent. +- **`AllNodesFailedException` groups a hostname's failures.** A contact point that resolves to + several addresses contributes one entry per contact point rather than per address, with each + address's failure attached to it as a suppressed exception. If you inspect + `AllNodesFailedException.getAllErrors()`, expect fewer entries carrying more detail; the underlying + causes are all still reachable through `Throwable.getSuppressed()`. +- **A name's addresses are tried in random order, and at most + `advanced.connection.max-candidate-addresses` of them per connection attempt** (new option, + default 5). The shuffle spreads load across a multi-record name's addresses and varies the + starting address between attempts. It applies to every contact point, and to an already-identified + node when its addresses are interchangeable — an SNI proxy or a cloud private-endpoint route, + where the proxy routes by server name so all of its addresses lead to the same node. For those the + driver spread connections across the records before this version too, inside + `SniEndPoint.resolve()`. The order is kept only for a name that may denote *different* hosts, + which is what an `AddressTranslator` can return (`SubnetAddressTranslator` does by default): one + node's connection pool converges on one address there, as it did before, and the remaining + addresses serve as fallback. The cap applies either way + and bounds what one attempt can cost, since every address + tried is a full TCP connect plus handshake — and, with wrong credentials, a rejected login + (authentication failures do not stop the walk: with a multi-record name they may be specific to + the address, e.g. a stale record pointing at a foreign cluster). Where the order is shuffled, + addresses beyond the cap are + not lost — every attempt samples a fresh random subset, so reconnection rounds reach all of them + over time; where it is kept, the cap is a hard limit and records past it are not reached, which is + still more than the single address such a name got before. Set the option to `1` to restore + one-address-per-attempt behavior. +- **The control connection can now decline a node after a successful handshake.** Once the driver + knows which node answered, it re-checks that node against what the load balancing policy and + topology events say about it, and closes the channel if it is IGNORED, forced down or removed, + moving on to the next entry in the plan. Previously a contact point reached this way was an + ephemeral object that no event had ever named, so the check found nothing and the connection was + kept. This matters most with the contact-point fallback above now on by default: if your contact + points resolve to nodes the policy excludes, expect the control connection to keep looking rather + than settle on one of them. The reason is recorded in the resulting `AllNodesFailedException`. +- If you use `TaggingMetricIdGenerator` **and** build the Cloud proxy address yourself with + `new InetSocketAddress("proxy-host", port)` rather than through a secure connect bundle: the + `node` tag for those nodes changes once, from `proxy-host/1.2.3.4:9042` to `proxy-host:9042`. The + driver now keeps the configured proxy address unresolved so it can be re-expanded to every proxy + A-record, and the tag no longer depends on which proxy IP a connection happened to land on — that + stability is the point, but expect a one-time series rename in dashboards. The same one-time + rename applies if you passed the proxy as an IP literal (`1.2.3.4/1.2.3.4:9042` becomes + `1.2.3.4:9042`), which is stored unresolved too so that the endpoint's identity cannot change + underneath it when something asks the address for its reverse-DNS name. `asMetricPrefix()`, and + therefore the default `MetricIdGenerator`, is unaffected. +- For advanced deployments that provide a custom `NettyOptions`: the + `afterBootstrapInitialized()` hook now runs once per logical connection to a node (previously + once per attempt, including protocol-version downgrade retries), and it receives the bootstrap + *before* the driver installs its channel handler — a handler set by the hook is replaced, and + the driver logs a one-time warning if it detects one. Use the hook for channel options, + attributes and `Bootstrap.resolver(...)`; use `afterChannelInitialized()` for pipeline + customization. +- Two `protected` methods that no longer have anything to do are removed. Both are in internal + packages, but they were reachable from a subclass, so a custom subclass overriding either of them + will no longer compile: + - `OptionalLocalDcHelper.checkLocalDatacenterCompatibility(String, Set)` — it warned when a + contact point's datacenter differed from the configured local DC, but contact-point nodes never + get a datacenter assigned, so it compared against `null` and warned unconditionally instead of + on a real mismatch. The separate "configured local DC matches no node" warning is retained and + now covers the intended case. + - `ClientRoutesTopologyMonitor.resolveAddress(String)` — a test seam for the JVM DNS lookup that + the monitor no longer performs, now that route hostnames are resolved by the connection layer. + ### 4.19.0.7 #### Cloud private-endpoint support via client routes