diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java index b7ba0b025cf2..3a8a19062137 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java @@ -17,11 +17,15 @@ package org.apache.hadoop.hdds.scm.container.balancer; +import static org.apache.hadoop.util.StringUtils.byteDesc; + import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -29,6 +33,7 @@ import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; /** * Orchestrates cluster analysis, estimation and recommendation for container balancer. @@ -36,6 +41,7 @@ public final class ContainerBalancerAdvisor { private static final long MIN_DELETE_PHASE_MILLIS = Duration.ofMinutes(9).toMillis(); + private static final double PLANNING_ITERATION_BUFFER = 1.3d; private static final String DATANODE_OFFSET_KEY = "hdds.scm.replication.event.timeout.datanode.offset"; @@ -82,6 +88,187 @@ public static List estimate(OzoneConfiguration conf return Collections.unmodifiableList(estimations); } + /** + * Recommends balancer configuration for one or more balancer profiles. + * + *

If {@link AdvisorRequest#profile} is set, returns a result for that profile only. + * Otherwise returns SLOW, MEDIUM, and FAST. Per-profile validation failures are returned with + * {@link ContainerBalancerRecommendation#succeeded()} false instead of aborting other profiles. + */ + public static List recommend( + OzoneConfiguration conf, AdvisorRequest request) { + Objects.requireNonNull(conf, "conf"); + Objects.requireNonNull(request, "request"); + List nodes = Objects.requireNonNull(request.nodes, "nodes"); + + ContainerBalancerConfiguration balancerConfig = conf.getObject(ContainerBalancerConfiguration.class); + + double thresholdPercent = request.thresholdPercent != null + ? request.thresholdPercent + : balancerConfig.getThreshold(); + validateThresholdPercent(thresholdPercent); + double thresholdRatio = thresholdPercent / 100.0; + Set includeNodes = request.includeNodes != null + ? request.includeNodes + : balancerConfig.getIncludeNodes(); + Set excludeNodes = request.excludeNodes != null + ? request.excludeNodes + : balancerConfig.getExcludeNodes(); + + ContainerBalancerClusterSnapshot snapshot = ContainerBalancerClusterAnalyzer.analyze( + nodes, thresholdRatio, includeNodes, excludeNodes); + + List profiles = selectProfilesForRecommend(request); + if (isClusterAlreadyBalanced(snapshot)) { + return recommendationsForBalancedCluster(profiles, thresholdPercent); + } + validateSnapshotForEstimation(snapshot); + List recommendations = new ArrayList<>(profiles.size()); + for (ContainerBalancerProfile profile : profiles) { + recommendations.add(recommendForProfile( + conf, request, profile, snapshot, balancerConfig, thresholdPercent)); + } + return Collections.unmodifiableList(recommendations); + } + + private static boolean isClusterAlreadyBalanced(ContainerBalancerClusterSnapshot snapshot) { + return snapshot.getBytesToMove() <= 0 + || (snapshot.getSourceCount() < 1 && snapshot.getTargetCount() < 1); + } + + private static List recommendationsForBalancedCluster( + List profiles, double thresholdPercent) { + List recommendations = new ArrayList<>(profiles.size()); + for (ContainerBalancerProfile profile : profiles) { + recommendations.add(ContainerBalancerRecommendation.newBuilder() + .setProfile(profile) + .setThresholdPercent(thresholdPercent) + .setClusterBalanced(true) + .build()); + } + return Collections.unmodifiableList(recommendations); + } + + private static ContainerBalancerRecommendation recommendForProfile( + OzoneConfiguration conf, + AdvisorRequest request, + ContainerBalancerProfile profile, + ContainerBalancerClusterSnapshot snapshot, + ContainerBalancerConfiguration balancerConfig, + double thresholdPercent) { + + ContainerBalancerRecommendation.Builder builder = ContainerBalancerRecommendation.newBuilder() + .setProfile(profile) + .setThresholdPercent(thresholdPercent); + + try { + long moveReplicationTimeoutMillis = balancerConfig.getMoveReplicationTimeout().toMillis(); + long moveTimeoutMillis = balancerConfig.getMoveTimeout().toMillis(); + long balancingIntervalMillis = balancerConfig.getBalancingInterval().toMillis(); + validateMoveTimeouts(conf, moveReplicationTimeoutMillis, moveTimeoutMillis); + validateBalancingIntervalMillis(balancingIntervalMillis); + + ContainerBalancerEstimation estimation = estimateForProfile( + conf, request, profile, snapshot, balancerConfig, thresholdPercent); + if (!estimation.succeeded()) { + throw new IllegalArgumentException(estimation.getFailureMessage()); + } + + long configCeiling = balancerConfig.getMaxSizeToMovePerIteration(); + long recommendedMaxMove = computeRecommendedMaxSizeToMove(configCeiling, estimation); + + validateResolvedMoveLimits( + conf, + estimation.getMaxSizeEnteringTarget(), + estimation.getMaxSizeLeavingSource(), + recommendedMaxMove); + + int recommendedIterations = (int) Math.ceil( + estimation.getEstimatedIterations() * PLANNING_ITERATION_BUFFER); + + Map rationale = buildRationale( + request.thresholdPercent != null, + profile, + balancerConfig, + estimation, + moveTimeoutMillis, + moveReplicationTimeoutMillis, + balancingIntervalMillis); + + return builder + .setMaxDatanodesPercentage(estimation.getMaxDatanodesPercentage()) + .setMaxSizeToMovePerIteration(recommendedMaxMove) + .setMaxSizeEnteringTarget(estimation.getMaxSizeEnteringTarget()) + .setMaxSizeLeavingSource(estimation.getMaxSizeLeavingSource()) + .setMoveTimeoutMillis(moveTimeoutMillis) + .setMoveReplicationTimeoutMillis(moveReplicationTimeoutMillis) + .setBalancingIntervalMillis(balancingIntervalMillis) + .setRecommendedIterations(recommendedIterations) + .setRationale(rationale) + .setEstimation(estimation) + .build(); + } catch (IllegalArgumentException e) { + return builder.setFailureMessage(e.getMessage()).build(); + } + } + + /** + * Recommended max-size-to-move for one profile: + * min(config ceiling, max(per-iteration estimate, profile entering, profile leaving)). + */ + static long computeRecommendedMaxSizeToMove( + long configCeiling, ContainerBalancerEstimation estimation) { + long floor = maxPositive( + estimation.getPerIterationBytes(), + estimation.getMaxSizeEnteringTarget(), + estimation.getMaxSizeLeavingSource()); + return minPositive(configCeiling, floor); + } + + private static Map buildRationale( + boolean userProvidedThreshold, + ContainerBalancerProfile profile, + ContainerBalancerConfiguration balancerConfig, + ContainerBalancerEstimation estimation, + long moveTimeoutMillis, + long moveReplicationTimeoutMillis, + long balancingIntervalMillis) { + Map rationale = new HashMap<>(); + rationale.put("threshold", userProvidedThreshold + ? "user override" + : "default from configuration"); + int profileDatanodesPercentage = profile.getDatanodesMaxPercentage(); + int usedDatanodesPercentage = estimation.getMaxDatanodesPercentage(); + if (usedDatanodesPercentage == profileDatanodesPercentage) { + rationale.put("maxDatanodesPercentage", String.format( + "profile default (%d%%)", usedDatanodesPercentage)); + } else { + rationale.put("maxDatanodesPercentage", String.format( + "profile %d%%, raised to %d%% (≥2 nodes)", profileDatanodesPercentage, usedDatanodesPercentage)); + } + rationale.put("maxSizeToMovePerIteration", String.format( + "min(%s ceiling, max(per-iteration %s, entering %s, leaving %s))", + byteDesc(balancerConfig.getMaxSizeToMovePerIteration()), + byteDesc(estimation.getPerIterationBytes()), + byteDesc(estimation.getMaxSizeEnteringTarget()), + byteDesc(estimation.getMaxSizeLeavingSource()))); + rationale.put("maxSizeEnteringTarget", String.format( + "profile default (%d GB)", estimation.getMaxSizeEnteringTarget() / OzoneConsts.GB)); + rationale.put("maxSizeLeavingSource", String.format( + "profile default (%d GB)", estimation.getMaxSizeLeavingSource() / OzoneConsts.GB)); + rationale.put("moveTimeout", String.format( + "configuration default (%d min)", moveTimeoutMillis / 60000)); + rationale.put("moveReplicationTimeout", String.format( + "configuration default (%d min)", moveReplicationTimeoutMillis / 60000)); + rationale.put("balancingInterval", String.format( + "configuration default (%d min)", balancingIntervalMillis / 60000)); + long planningIterations = (long) Math.ceil( + estimation.getEstimatedIterations() * PLANNING_ITERATION_BUFFER); + rationale.put("iterations", String.format( + "includes +30%% buffer (planning estimate: %d)", planningIterations)); + return rationale; + } + private static ContainerBalancerEstimation estimateForProfile(OzoneConfiguration conf, AdvisorRequest request, ContainerBalancerProfile profile, ContainerBalancerClusterSnapshot snapshot, ContainerBalancerConfiguration balancerConfig, double thresholdPercent) { @@ -167,6 +354,7 @@ private static ContainerBalancerEstimation estimateForProfile(OzoneConfiguration .build(); } catch (IllegalArgumentException e) { return builder + .setThresholdPercent(thresholdPercent) .setMaxDatanodesPercentage(maxDatanodesPercentage) .setFailureMessage(e.getMessage()) .build(); @@ -245,7 +433,17 @@ private static long minPositive(long... values) { return result; } - private static void validateSnapshotForEstimation(ContainerBalancerClusterSnapshot snapshot) { + private static long maxPositive(long... values) { + long result = 0; + for (long value : values) { + if (value > result) { + result = value; + } + } + return result; + } + + private static void validateSnapshotForEstimation(ContainerBalancerClusterSnapshot snapshot ) { if (snapshot.getSourceCount() < 1) { throw new IllegalArgumentException("No over-utilized datanodes (sources) found."); } @@ -351,6 +549,16 @@ private static List selectProfiles(AdvisorRequest requ return Collections.singletonList(ContainerBalancerProfile.MEDIUM); } + private static List selectProfilesForRecommend(AdvisorRequest request) { + if (request.profile != null) { + return Collections.singletonList(request.profile); + } + return Arrays.asList( + ContainerBalancerProfile.SLOW, + ContainerBalancerProfile.MEDIUM, + ContainerBalancerProfile.FAST); + } + /** * Input for {@link ContainerBalancerAdvisor}: cluster usage data and optional overrides. * Unset fields fall back to {@link ContainerBalancerConfiguration} or profile presets. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerRecommendation.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerRecommendation.java new file mode 100644 index 000000000000..bcd3197a57e0 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerRecommendation.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +/** + * Recommended balancer configuration for a single profile. + */ +public final class ContainerBalancerRecommendation { + + private final ContainerBalancerProfile profile; + private final String failureMessage; + private final double thresholdPercent; + private final int maxDatanodesPercentage; + private final long maxSizeToMovePerIteration; + private final long maxSizeEnteringTarget; + private final long maxSizeLeavingSource; + private final long moveTimeoutMillis; + private final long moveReplicationTimeoutMillis; + private final long balancingIntervalMillis; + private final int recommendedIterations; + private final Map rationale; + private final ContainerBalancerEstimation estimation; + private final boolean clusterBalanced; + + private ContainerBalancerRecommendation(Builder b) { + this.profile = Objects.requireNonNull(b.profile, "profile == null"); + this.failureMessage = b.failureMessage; + this.thresholdPercent = b.thresholdPercent; + this.maxDatanodesPercentage = b.maxDatanodesPercentage; + this.maxSizeToMovePerIteration = b.maxSizeToMovePerIteration; + this.maxSizeEnteringTarget = b.maxSizeEnteringTarget; + this.maxSizeLeavingSource = b.maxSizeLeavingSource; + this.moveTimeoutMillis = b.moveTimeoutMillis; + this.moveReplicationTimeoutMillis = b.moveReplicationTimeoutMillis; + this.balancingIntervalMillis = b.balancingIntervalMillis; + this.recommendedIterations = b.recommendedIterations; + this.rationale = b.rationale == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(b.rationale); + this.estimation = b.estimation; + this.clusterBalanced = b.clusterBalanced; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public ContainerBalancerProfile getProfile() { + return profile; + } + + public boolean succeeded() { + return failureMessage == null; + } + + public boolean isClusterBalanced() { + return clusterBalanced; + } + + public String getFailureMessage() { + return failureMessage; + } + + public double getThresholdPercent() { + return thresholdPercent; + } + + public int getMaxDatanodesPercentage() { + return maxDatanodesPercentage; + } + + public long getMaxSizeToMovePerIteration() { + return maxSizeToMovePerIteration; + } + + public long getMaxSizeEnteringTarget() { + return maxSizeEnteringTarget; + } + + public long getMaxSizeLeavingSource() { + return maxSizeLeavingSource; + } + + public long getMoveTimeoutMillis() { + return moveTimeoutMillis; + } + + public long getMoveReplicationTimeoutMillis() { + return moveReplicationTimeoutMillis; + } + + public long getBalancingIntervalMillis() { + return balancingIntervalMillis; + } + + public int getRecommendedIterations() { + return recommendedIterations; + } + + public Map getRationale() { + return rationale; + } + + public ContainerBalancerEstimation getEstimation() { + return estimation; + } + + /** Builder for {@link ContainerBalancerRecommendation}. */ + public static final class Builder { + private ContainerBalancerProfile profile; + private String failureMessage; + private double thresholdPercent; + private int maxDatanodesPercentage; + private long maxSizeToMovePerIteration; + private long maxSizeEnteringTarget; + private long maxSizeLeavingSource; + private long moveTimeoutMillis; + private long moveReplicationTimeoutMillis; + private long balancingIntervalMillis; + private int recommendedIterations; + private Map rationale; + private ContainerBalancerEstimation estimation; + private boolean clusterBalanced; + + private Builder() { + } + + public Builder setProfile(ContainerBalancerProfile profileValue) { + this.profile = profileValue; + return this; + } + + public Builder setFailureMessage(String message) { + this.failureMessage = message; + return this; + } + + public Builder setThresholdPercent(double threshold) { + this.thresholdPercent = threshold; + return this; + } + + public Builder setMaxDatanodesPercentage(int percentage) { + this.maxDatanodesPercentage = percentage; + return this; + } + + public Builder setMaxSizeToMovePerIteration(long bytes) { + this.maxSizeToMovePerIteration = bytes; + return this; + } + + public Builder setMaxSizeEnteringTarget(long bytes) { + this.maxSizeEnteringTarget = bytes; + return this; + } + + public Builder setMaxSizeLeavingSource(long bytes) { + this.maxSizeLeavingSource = bytes; + return this; + } + + public Builder setMoveTimeoutMillis(long millis) { + this.moveTimeoutMillis = millis; + return this; + } + + public Builder setMoveReplicationTimeoutMillis(long millis) { + this.moveReplicationTimeoutMillis = millis; + return this; + } + + public Builder setBalancingIntervalMillis(long millis) { + this.balancingIntervalMillis = millis; + return this; + } + + public Builder setRecommendedIterations(int iterations) { + this.recommendedIterations = iterations; + return this; + } + + public Builder setRationale(Map rationaleMap) { + this.rationale = rationaleMap; + return this; + } + + public Builder setEstimation(ContainerBalancerEstimation estimationValue) { + this.estimation = estimationValue; + return this; + } + + public Builder setClusterBalanced(boolean balanced) { + this.clusterBalanced = balanced; + return this; + } + + public ContainerBalancerRecommendation build() { + return new ContainerBalancerRecommendation(this); + } + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java index 411183f45048..46cb5933f955 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java @@ -20,7 +20,10 @@ import static org.apache.hadoop.ozone.ClientVersion.DEFAULT_VERSION; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.List; @@ -34,6 +37,19 @@ /** Tests for {@link ContainerBalancerAdvisor} estimation. */ public final class TestContainerBalancerAdvisor { + @Test + void testComputePerIterationBytesLimitedByEnteringTarget() { + int[] involved = {7, 7}; + long expected = 26L * OzoneConsts.GB * 7; + + assertEquals(expected, ContainerBalancerAdvisor.computePerIterationBytes( + expected * 10, + 500L * OzoneConsts.GB, + 26 * OzoneConsts.GB, + 26 * OzoneConsts.GB, + involved)); + } + @Test void testComputePerIterationBytesNeverExceedsBytesToMove() { int[] involved = {3, 3}; @@ -240,6 +256,124 @@ void testEstimateFailsWhenClusterBalanced() { new ContainerBalancerAdvisor.AdvisorRequest().setNodes(balanced))); } + @Test + void testRecommendReturnsThreeProfiles() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(buildCluster(70, 14, 14))); + + assertEquals(3, results.size()); + assertEquals(ContainerBalancerProfile.SLOW, results.get(0).getProfile()); + assertEquals(ContainerBalancerProfile.MEDIUM, results.get(1).getProfile()); + assertEquals(ContainerBalancerProfile.FAST, results.get(2).getProfile()); + for (ContainerBalancerRecommendation result : results) { + assertTrue(result.succeeded()); + assertNotNull(result.getEstimation()); + assertTrue(result.getEstimation().succeeded()); + assertTrue(result.getRecommendedIterations() >= result.getEstimation().getEstimatedIterations()); + assertFalse(result.getRationale().isEmpty()); + } + } + + @Test + void testComputeRecommendedMaxSizeToMoveFloorsAtProfileLimits() { + ContainerBalancerEstimation estimation = ContainerBalancerEstimation.newBuilder() + .setProfile(ContainerBalancerProfile.SLOW) + .setPerIterationBytes(8L * OzoneConsts.GB) + .setMaxSizeEnteringTarget(10L * OzoneConsts.GB) + .setMaxSizeLeavingSource(10L * OzoneConsts.GB) + .build(); + + assertEquals(10L * OzoneConsts.GB, + ContainerBalancerAdvisor.computeRecommendedMaxSizeToMove(500L * OzoneConsts.GB, estimation)); + } + + @Test + void testComputeRecommendedMaxSizeToMoveUsesPerIterationWhenLarger() { + ContainerBalancerEstimation estimation = ContainerBalancerEstimation.newBuilder() + .setProfile(ContainerBalancerProfile.MEDIUM) + .setPerIterationBytes(182L * OzoneConsts.GB) + .setMaxSizeEnteringTarget(26L * OzoneConsts.GB) + .setMaxSizeLeavingSource(26L * OzoneConsts.GB) + .build(); + + assertEquals(182L * OzoneConsts.GB, + ContainerBalancerAdvisor.computeRecommendedMaxSizeToMove(500L * OzoneConsts.GB, estimation)); + } + + @Test + void testRecommendReturnsSingleProfileWhenProfileSet() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setProfile(ContainerBalancerProfile.MEDIUM)); + + assertEquals(1, results.size()); + assertEquals(ContainerBalancerProfile.MEDIUM, results.get(0).getProfile()); + assertTrue(results.get(0).succeeded()); + } + + @Test + void testRecommendUsesProfileDefaults() { + OzoneConfiguration conf = new OzoneConfiguration(); + ContainerBalancerRecommendation slow = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(buildCluster(70, 14, 14))) + .get(0); + + assertEquals(10, slow.getMaxDatanodesPercentage()); + assertEquals(10L * OzoneConsts.GB, slow.getMaxSizeEnteringTarget()); + assertEquals(10L * OzoneConsts.GB, slow.getMaxSizeLeavingSource()); + assertEquals(30L * OzoneConsts.GB, slow.getMaxSizeToMovePerIteration()); + assertTrue(slow.getMaxSizeToMovePerIteration() >= slow.getMaxSizeEnteringTarget()); + assertTrue(slow.getMaxSizeToMovePerIteration() >= slow.getMaxSizeLeavingSource()); + } + + @Test + void testRecommendRespectsThresholdOverride() { + OzoneConfiguration conf = new OzoneConfiguration(); + List nodes = buildCluster(70, 14, 14); + + long defaultBytesToMove = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(nodes)) + .get(0) + .getEstimation() + .getBytesToMove(); + + long tighterBytesToMove = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(nodes) + .setThresholdPercent(5.0)) + .get(0) + .getEstimation() + .getBytesToMove(); + + assertTrue(tighterBytesToMove > defaultBytesToMove); + } + + @Test + void testRecommendSucceedsWhenClusterBalanced() { + OzoneConfiguration conf = new OzoneConfiguration(); + List balanced = new ArrayList<>(); + balanced.add(proto("dn-1", OzoneConsts.TB, (long) (0.70 * OzoneConsts.TB))); + balanced.add(proto("dn-2", OzoneConsts.TB, (long) (0.70 * OzoneConsts.TB))); + + List results = ContainerBalancerAdvisor.recommend( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(balanced)); + + assertEquals(3, results.size()); + for (ContainerBalancerRecommendation result : results) { + assertThat(result.succeeded()).isTrue(); + assertThat(result.isClusterBalanced()).isTrue(); + } + } + @Test void testEstimateFailsWhenEnteringTargetTooSmall() { OzoneConfiguration conf = new OzoneConfiguration(); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCliHelper.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCliHelper.java new file mode 100644 index 000000000000..3cb8961959a9 --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCliHelper.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli; + +import static org.apache.hadoop.util.StringUtils.byteDesc; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerEstimation; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerProfile; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerRecommendation; +import org.apache.hadoop.ozone.OzoneConsts; + +/** Shared parsing and CLI output for container balancer subcommands. */ +public final class ContainerBalancerCliHelper { + + /** Multiplier for recommended/planning iteration counts and CLI planning display (+30%). */ + public static final double PLANNING_ITERATION_BUFFER = 1.3d; + + private ContainerBalancerCliHelper() { + } + + public static ContainerBalancerProfile parseProfile(String name) throws IOException { + try { + return ContainerBalancerProfile.valueOf(name.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid profile: " + name + ". Expected SLOW, MEDIUM, or FAST."); + } + } + + public static Set parseNodeSet(String nodes) { + if (StringUtils.isBlank(nodes)) { + return Collections.emptySet(); + } + return Arrays.stream(nodes.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); + } + + public static void printEstimation(PrintWriter out, ContainerBalancerEstimation estimation) { + long estimatedIterations = estimation.getEstimatedIterations(); + long planningIterations = (long) Math.ceil(estimatedIterations * PLANNING_ITERATION_BUFFER); + long cycleTimeMillis = estimation.getMoveTimeoutMillis() + estimation.getBalancingIntervalMillis(); + long baseDurationMillis = estimation.getEstimatedDurationMillis(); + long planningDurationMillis = planningIterations * cycleTimeMillis; + out.printf(" Bytes to move: %s%n", byteDesc(estimation.getBytesToMove())); + out.printf(" Per iteration (estimate): ~%s%n", byteDesc(estimation.getPerIterationBytes())); + out.printf(" Estimated iterations: %d (planning estimate: %d, includes +30%% buffer)%n", + estimatedIterations, planningIterations); + out.printf(" Estimated duration: upper bound %s (planning estimate: %s, includes +30%% buffer)%n", + formatEstimatedDuration(baseDurationMillis), + formatEstimatedDuration(planningDurationMillis)); + out.println(" (assumes full move timeout + interval each cycle)"); + out.println(); + } + + public static String formatEstimatedDuration(long durationMillis) { + double days = durationMillis / 86400000d; + if (days >= 1) { + return String.format("~%.1f days", days); + } + double hours = durationMillis / 3600000d; + if (hours >= 1) { + return String.format("~%.1f hours", hours); + } + long minutes = durationMillis / 60000; + return String.format("~%d min", minutes); + } + + public static String formatStartCommand( + ContainerBalancerRecommendation recommendation, + Optional includeNodes, + Optional excludeNodes) { + StringBuilder sb = new StringBuilder("ozone admin containerbalancer start"); + sb.append(" -t ").append(formatThreshold(recommendation.getThresholdPercent())); + sb.append(" -i ").append(recommendation.getRecommendedIterations()); + sb.append(" -d ").append(recommendation.getMaxDatanodesPercentage()); + sb.append(" -s ").append(bytesToGb(recommendation.getMaxSizeToMovePerIteration())); + sb.append(" -e ").append(bytesToGb(recommendation.getMaxSizeEnteringTarget())); + sb.append(" -l ").append(bytesToGb(recommendation.getMaxSizeLeavingSource())); + sb.append(" --balancing-iteration-interval-minutes ") + .append(recommendation.getBalancingIntervalMillis() / 60000); + sb.append(" --move-timeout-minutes ") + .append(recommendation.getMoveTimeoutMillis() / 60000); + sb.append(" --move-replication-timeout-minutes ") + .append(recommendation.getMoveReplicationTimeoutMillis() / 60000); + appendNodeFilters(sb, includeNodes, excludeNodes); + return sb.toString(); + } + + private static void appendNodeFilters( + StringBuilder sb, + Optional includeNodes, + Optional excludeNodes) { + includeNodes.filter(s -> !StringUtils.isBlank(s)) + .ifPresent(s -> sb.append(" --include-datanodes ").append(s.trim())); + excludeNodes.filter(s -> !StringUtils.isBlank(s)) + .ifPresent(s -> sb.append(" --exclude-datanodes ").append(s.trim())); + } + + private static long bytesToGb(long bytes) { + return bytes / OzoneConsts.GB; + } + + private static String formatThreshold(double thresholdPercent) { + if (thresholdPercent == Math.rint(thresholdPercent)) { + return String.valueOf((long) thresholdPercent); + } + return String.valueOf(thresholdPercent); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index c65cb1f4ca1b..2ca6eaf0c207 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -54,6 +54,11 @@ * involved in balancing * ozone admin containerbalancer start -s 10 * start balancer with maximum size of 10GB to move in one iteration + * To assess: + * ozone admin containerbalancer assessment + * [ -t/--threshold {@literal }] + * [ --include-datanodes {@literal }] + * [ --exclude-datanodes {@literal }] * To estimate: * ozone admin containerbalancer estimate * [ --profile {@literal } ] @@ -78,21 +83,28 @@ * estimate for the SLOW profile only * ozone admin containerbalancer estimate --profile fast -t 5 * estimate FAST profile with a 5% threshold + * To recommend: + * ozone admin containerbalancer recommend + * [ --profile {@literal } ] + * [ -t/--threshold {@literal } ] + * [ --include-datanodes {@literal } ] + * [ --exclude-datanodes {@literal } ] + * Examples: + * ozone admin containerbalancer recommend + * recommend balancer configuration for SLOW, MEDIUM, and FAST profiles + * ozone admin containerbalancer recommend --profile slow + * recommend for the SLOW profile only + * ozone admin containerbalancer recommend -t 5 + * recommend with a 5% threshold * To stop: * ozone admin containerbalancer stop - * To assess: - * ozone admin containerbalancer assessment - * [ -t/--threshold {@literal }] - * [ --include-datanodes {@literal }] - * [ --exclude-datanodes {@literal }] * * *

DESCRIPTION - *

The estimate subcommand fetches datanode usage from SCM and estimates from - * local configurations, profile presets and cluster analysis made. It does not start the balancer. Start does not yet - * support {@code --profile}, compare estimate profiles to the config you plan - * to pass on start, or wait until profile support is added to start. - * estimate subcommand produces upper-bound estimates. + *

Estimate and recommend fetch datanode usage from SCM and use local configuration and cluster + * analysis. They do not start the balancer. Start does not yet support {@code --profile}; compare + * estimate/recommend profiles to the config you plan to pass on start. Estimate produces upper-bound + * duration estimates. *

The threshold parameter is a fraction in the range of (1%, 100%) with a * default value of 10%. The threshold sets a target for whether the cluster * is balanced. A cluster is balanced if for each datanode, the utilization @@ -118,7 +130,8 @@ ContainerBalancerStopSubcommand.class, ContainerBalancerStatusSubcommand.class, ContainerBalancerAssessmentSubcommand.class, - ContainerBalancerEstimateSubcommand.class + ContainerBalancerEstimateSubcommand.class, + ContainerBalancerRecommendSubcommand.class }) @MetaInfServices(AdminSubcommand.class) public class ContainerBalancerCommands implements AdminSubcommand { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java index e82a954f2fd3..eb0a7bd0bb51 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java @@ -18,12 +18,7 @@ package org.apache.hadoop.hdds.scm.cli; import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerAdvisor; import org.apache.hadoop.ozone.OzoneConsts; import picocli.CommandLine.Option; @@ -159,17 +154,7 @@ public void applyToEstimateRequest(ContainerBalancerAdvisor.AdvisorRequest reque request.setMoveTimeoutMillis(Duration.ofMinutes(minutes).toMillis())); moveReplicationTimeout.ifPresent(minutes -> request.setMoveReplicationTimeoutMillis(Duration.ofMinutes(minutes).toMillis())); - includeNodes.ifPresent(value -> request.setIncludeNodes(parseNodeSet(value))); - excludeNodes.ifPresent(value -> request.setExcludeNodes(parseNodeSet(value))); - } - - private static Set parseNodeSet(String nodes) { - if (StringUtils.isBlank(nodes)) { - return Collections.emptySet(); - } - return Arrays.stream(nodes.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toSet()); + includeNodes.ifPresent(value -> request.setIncludeNodes(ContainerBalancerCliHelper.parseNodeSet(value))); + excludeNodes.ifPresent(value -> request.setExcludeNodes(ContainerBalancerCliHelper.parseNodeSet(value))); } } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerEstimateSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerEstimateSubcommand.java index 3e0b984b0508..8278080ce2be 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerEstimateSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerEstimateSubcommand.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.Locale; import java.util.Optional; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -29,7 +28,6 @@ import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerAdvisor; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerEstimation; -import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerProfile; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -47,8 +45,6 @@ versionProvider = HddsVersionProvider.class) public class ContainerBalancerEstimateSubcommand extends ScmSubcommand { - private static final double PLANNING_ITERATION_BUFFER = 1.3d; - @CommandLine.Mixin private ContainerBalancerConfigOptions configOptions; @@ -83,7 +79,7 @@ public void execute(ScmClient scmClient) throws IOException { printBasedOn(result); if (result.succeeded()) { anySucceeded = true; - printEstimation(result); + ContainerBalancerCliHelper.printEstimation(out(), result); } else if (multipleProfiles) { out().printf(" Estimation failed: %s%n%n", result.getFailureMessage()); } @@ -104,26 +100,18 @@ private ContainerBalancerAdvisor.AdvisorRequest buildRequest(List= 1) { - return String.format(Locale.ENGLISH, "~%.1f days", days); - } - double hours = durationMillis / 3600000d; - if (hours >= 1) { - return String.format(Locale.ENGLISH, "~%.1f hours", hours); - } - long minutes = durationMillis / 60000; - return String.format(Locale.ENGLISH, "~%d min", minutes); - } - static class ProfileSelection { @Option(names = {"--profile"}, description = "Throttling profile: SLOW, MEDIUM, or FAST profiles. When set, only this profile is estimated. " diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerRecommendSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerRecommendSubcommand.java new file mode 100644 index 000000000000..46ce3da80eef --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerRecommendSubcommand.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli; + +import static org.apache.hadoop.util.StringUtils.byteDesc; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerAdvisor; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerRecommendation; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** + * Recommends container balancer configuration for SLOW, MEDIUM, and FAST profiles + * without starting the balancer. + */ +@Command( + name = "recommend", + description = "Recommend container balancer configuration based on current cluster imbalance. " + + "When --profile is omitted, SLOW, MEDIUM, and FAST are recommended. Does not start the balancer.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class ContainerBalancerRecommendSubcommand extends ScmSubcommand { + + private static final int PARAM_COLUMN_WIDTH = 42; + private static final int VALUE_COLUMN_WIDTH = 14; + + @Option(names = {"-t", "--threshold"}, + description = "Percentage deviation from average utilization of " + + "the cluster after which a datanode will be rebalanced. The value " + + "should be in the range [0.0, 100.0), with a default of 10 " + + "(specify '10' for 10%%).") + private Optional threshold; + + @Option(names = {"--include-datanodes"}, + description = "A list of Datanode hostnames or ip addresses separated by commas. Only the " + + "Datanodes specified in this list are balanced.") + private Optional includeNodes; + + @Option(names = {"--exclude-datanodes"}, + description = "A list of Datanode hostnames or ip addresses separated by commas. The " + + "Datanodes specified in this list are excluded from balancing.") + private Optional excludeNodes; + + @Option(names = {"--profile"}, + description = "Throttling profile: SLOW, MEDIUM, or FAST. When set, only this profile is " + + "recommended. When omitted, SLOW, MEDIUM, and FAST are recommended.") + private Optional profileName; + + @Override + public void execute(ScmClient scmClient) throws IOException { + List nodes = scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE); + if (nodes == null || nodes.isEmpty()) { + throw new IOException("No datanode usage information available from SCM."); + } + + OzoneConfiguration conf = getOzoneConf(); + ContainerBalancerAdvisor.AdvisorRequest request = buildRequest(nodes); + List recommendations; + try { + recommendations = ContainerBalancerAdvisor.recommend(conf, request); + } catch (IllegalArgumentException e) { + throw new IOException(e.getMessage(), e); + } + + if (!recommendations.isEmpty() + && recommendations.stream().allMatch(ContainerBalancerRecommendation::isClusterBalanced)) { + out().println("Cluster is already balanced within the configured threshold. " + + "No balancing recommended."); + return; + } + + boolean anySucceeded = false; + for (ContainerBalancerRecommendation recommendation : recommendations) { + printRecommendation(recommendation); + if (recommendation.succeeded()) { + anySucceeded = true; + } + } + if (!anySucceeded) { + throw new IOException(recommendations.get(0).getFailureMessage()); + } + } + + private ContainerBalancerAdvisor.AdvisorRequest buildRequest(List nodes) + throws IOException { + ContainerBalancerAdvisor.AdvisorRequest request = + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(nodes); + threshold.ifPresent(request::setThresholdPercent); + includeNodes.ifPresent(value -> request.setIncludeNodes(ContainerBalancerCliHelper.parseNodeSet(value))); + excludeNodes.ifPresent(value -> request.setExcludeNodes(ContainerBalancerCliHelper.parseNodeSet(value))); + if (profileName.isPresent()) { + request.setProfile(ContainerBalancerCliHelper.parseProfile(profileName.get())); + } + return request; + } + + private void printRecommendation(ContainerBalancerRecommendation recommendation) { + out().printf("RECOMMENDED CONFIGURATION (profile: %s)%n", recommendation.getProfile().name()); + out().println(); + + if (!recommendation.succeeded()) { + out().printf("Recommendation failed: %s%n%n", recommendation.getFailureMessage()); + return; + } + + printRecommendedParameters(recommendation); + out().println(); + out().println(" Estimation:"); + ContainerBalancerCliHelper.printEstimation(out(), recommendation.getEstimation()); + + out().println(" Suggested commands:"); + out().println(" " + ContainerBalancerCliHelper.formatStartCommand( + recommendation, includeNodes, excludeNodes)); + out().println(); + } + + private void printRecommendedParameters(ContainerBalancerRecommendation recommendation) { + Map rationale = recommendation.getRationale(); + long moveTimeoutMinutes = Math.round(recommendation.getMoveTimeoutMillis() / 60000d); + long moveReplicationTimeoutMinutes = + Math.round(recommendation.getMoveReplicationTimeoutMillis() / 60000d); + long balancingIntervalMinutes = Math.round(recommendation.getBalancingIntervalMillis() / 60000d); + + out().println(" Recommended parameters:"); + printParameterRow("Parameter", "Value", "Rationale"); + printParameterRow("--threshold", + String.format("%.1f%%", recommendation.getThresholdPercent()), + rationale.get("threshold")); + printParameterRow("--max-datanodes-percentage-to-involve", + String.format("%d%%", recommendation.getMaxDatanodesPercentage()), + rationale.get("maxDatanodesPercentage")); + printParameterRow("--max-size-to-move-per-iteration-in-gb", + byteDesc(recommendation.getMaxSizeToMovePerIteration()), + rationale.get("maxSizeToMovePerIteration")); + printParameterRow("--max-size-entering-target-in-gb", + byteDesc(recommendation.getMaxSizeEnteringTarget()) + " / node", + rationale.get("maxSizeEnteringTarget")); + printParameterRow("--max-size-leaving-source-in-gb", + byteDesc(recommendation.getMaxSizeLeavingSource()) + " / node", + rationale.get("maxSizeLeavingSource")); + printParameterRow("--move-timeout-minutes", + String.format("%d min", moveTimeoutMinutes), + rationale.get("moveTimeout")); + printParameterRow("--move-replication-timeout-minutes", + String.format("%d min", moveReplicationTimeoutMinutes), + rationale.get("moveReplicationTimeout")); + printParameterRow("--balancing-iteration-interval-minutes", + String.format("%d min", balancingIntervalMinutes), + rationale.get("balancingInterval")); + printParameterRow("--iterations", + String.valueOf(recommendation.getRecommendedIterations()), + rationale.get("iterations")); + } + + private void printParameterRow(String parameter, String value, String rationaleText) { + String rationale = rationaleText == null ? "" : rationaleText; + out().printf(" %-" + PARAM_COLUMN_WIDTH + "s %-" + VALUE_COLUMN_WIDTH + "s %s%n", + parameter, value, rationale); + } + +} diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestContainerBalancerSubCommand.java index 70cce33a3e38..8a7a24d6df26 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/TestContainerBalancerSubCommand.java @@ -167,6 +167,7 @@ class TestContainerBalancerSubCommand { private ContainerBalancerStartSubcommand startCmd; private ContainerBalancerStatusSubcommand statusCmd; private ContainerBalancerEstimateSubcommand estimateCmd; + private ContainerBalancerRecommendSubcommand recommendCmd; private GenericTestUtils.PrintStreamCapturer out; private GenericTestUtils.PrintStreamCapturer err; private AtomicBoolean verbose; @@ -378,6 +379,7 @@ protected boolean isVerbose() { }; parseSubcommand(startCmd); estimateCmd = new ContainerBalancerEstimateSubcommand(); + recommendCmd = new ContainerBalancerRecommendSubcommand(); out = GenericTestUtils.captureOut(); err = GenericTestUtils.captureErr(); } @@ -1028,6 +1030,94 @@ void testContainerBalancerEstimateSubcommandPartialFailureWhenMaxMoveOverrideCon .doesNotContain("Per iteration (estimate):"); } + @Test + void testContainerBalancerRecommendSubcommandDefaultShowsAllProfiles() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(recommendCmd); + recommendCmd.execute(scmClient); + + String output = out.get(); + assertThat(output) + .contains("RECOMMENDED CONFIGURATION (profile: SLOW)") + .contains("RECOMMENDED CONFIGURATION (profile: MEDIUM)") + .contains("RECOMMENDED CONFIGURATION (profile: FAST)") + .contains("--threshold") + .contains("--iterations") + .contains(" Estimation:") + .contains("Bytes to move:") + .contains("planning estimate:") + .contains("assumes full move timeout + interval each cycle") + .contains("Suggested commands:") + .contains("ozone admin containerbalancer start") + .doesNotContain("Recommendation failed:"); + assertThat(output.split("Suggested commands:")).hasSize(4); + } + + @Test + void testContainerBalancerRecommendSubcommandWithThresholdOverride() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(recommendCmd, "-t", "5"); + recommendCmd.execute(scmClient); + + assertThat(out.get()) + .contains("RECOMMENDED CONFIGURATION (profile: SLOW)") + .contains("ozone admin containerbalancer start -t 5"); + } + + @Test + void testContainerBalancerRecommendSubcommandWithProfileShowsOneProfile() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(recommendCmd, "--profile", "medium"); + recommendCmd.execute(scmClient); + + String output = out.get(); + assertThat(output) + .contains("RECOMMENDED CONFIGURATION (profile: MEDIUM)") + .doesNotContain("RECOMMENDED CONFIGURATION (profile: SLOW)") + .doesNotContain("RECOMMENDED CONFIGURATION (profile: FAST)") + .contains("Suggested commands:") + .contains("ozone admin containerbalancer start"); + assertThat(output.split("Suggested commands:")).hasSize(2); + } + + @Test + void testContainerBalancerRecommendSubcommandInvalidThresholdFails() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(recommendCmd, "-t", "-1"); + IOException ex = assertThrows(IOException.class, () -> recommendCmd.execute(scmClient)); + assertThat(ex.getMessage()).contains("Threshold should be specified in the range [0.0, 100.0)."); + } + + @Test + void testContainerBalancerRecommendSubcommandWhenClusterBalanced() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + List balanced = new ArrayList<>(); + long capacity = OzoneConsts.TB; + balanced.add(datanodeUsageProto("dn-1", capacity, (long) (capacity * 0.70))); + balanced.add(datanodeUsageProto("dn-2", capacity, (long) (capacity * 0.70))); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)).thenReturn(balanced); + + parseSubcommand(recommendCmd); + recommendCmd.execute(scmClient); + + assertThat(out.get()) + .contains("Cluster is already balanced within the configured threshold.") + .doesNotContain("Suggested commands:") + .doesNotContain("ozone admin containerbalancer start"); + } + /** * Imbalanced cluster for estimate CLI tests. *