diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java index b2654e5203d7..bc3a18cd1b94 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java @@ -17,32 +17,54 @@ package org.apache.hadoop.ozone.admin.upgrade; -import java.io.IOException; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; -import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.admin.om.OmAddressOptions; +import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import picocli.CommandLine; /** * Sub command to query the overall upgrade status of the cluster, returning information about the finalization - * status of SCM, the datanodes and OM. + * status of SCM, the datanodes and OM. The command makes a single call to OM which returns the status of the other + * components as well as itself. */ @CommandLine.Command( name = "status", description = "Show status of the cluster upgrade", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) -public class StatusSubCommand extends ScmSubcommand { +public class StatusSubCommand extends AbstractSubcommand implements Callable { + + @CommandLine.Mixin + private OmAddressOptions.OptionalServiceIdOrHostMixin omAddressOptions; @Override - public void execute(ScmClient client) throws IOException { - HddsProtos.UpgradeStatus status = client.queryUpgradeStatus(); - - out().println("Upgrade status:"); - out().println(" SCM Finalized: " + status.getScmFinalized()); - out().println(" Datanodes finalized: " + status.getNumDatanodesFinalized()); - out().println(" Total Datanodes: " + status.getNumDatanodesTotal()); - out().println(" Should Finalize: " + status.getShouldFinalize()); + public Integer call() throws Exception { + try (OzoneManagerProtocol client = getClient()) { + OzoneManagerVersion omVersion = RpcClient.getOmVersion(client.getServiceInfo()); + if (!OzoneManagerVersion.ZDU.isSupportedBy(omVersion)) { + err().println("OM does not support zero downtime upgrade. The cluster upgrade status should be queried with " + + "`ozone admin scm finalizationstatus` and `ozone admin om finalizationstatus`"); + return 1; + } + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse status = client.queryUpgradeStatus(); + + + out().println("Upgrade status:"); + out().println(" OM Finalized? " + status.getOmFinalized()); + out().println(" SCM Finalized? " + status.getHddsStatus().getScmFinalized()); + out().println(" Datanodes finalized: " + status.getHddsStatus().getNumDatanodesFinalized() + + "/" + status.getHddsStatus().getNumDatanodesTotal()); + } + return 0; } + + protected OzoneManagerProtocol getClient() throws Exception { + return omAddressOptions.newClient(); + } + } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java index d0e0a439d1a0..26cfef1ffe63 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java @@ -17,18 +17,25 @@ package org.apache.hadoop.ozone.admin.upgrade; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; +import java.util.Collections; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.helpers.ServiceInfo; +import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -42,40 +49,85 @@ public class TestStatusSubCommand { private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + private OzoneManagerProtocol omClient; private StatusSubCommand cmd; @BeforeEach - public void setup() throws UnsupportedEncodingException { - cmd = new StatusSubCommand(); + public void setup() throws IOException { + omClient = mock(OzoneManagerProtocol.class); + when(omClient.getServiceInfo()).thenReturn(serviceInfoWithVersion(OzoneManagerVersion.ZDU)); + + cmd = new StatusSubCommand() { + @Override + protected OzoneManagerProtocol getClient() throws Exception { + return omClient; + } + }; System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); } @AfterEach public void tearDown() { System.setOut(originalOut); + System.setErr(originalErr); } @Test - public void testStatusCommandPrintsUpgradeStatus() throws IOException { - ScmClient scmClient = mock(ScmClient.class); - HddsProtos.UpgradeStatus status = HddsProtos.UpgradeStatus.newBuilder() + public void testStatusCommandPrintsUpgradeStatus() throws Exception { + HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() .setScmFinalized(false) .setNumDatanodesFinalized(1) .setNumDatanodesTotal(3) .setShouldFinalize(true) .build(); - when(scmClient.queryUpgradeStatus()).thenReturn(status); + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() + .setOmFinalized(false) + .setHddsStatus(hddsStatus) + .build(); + + when(omClient.queryUpgradeStatus()).thenReturn(response); new CommandLine(cmd).parseArgs(); - cmd.execute(scmClient); + cmd.call(); String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("Upgrade status:")); - assertTrue(output.contains("SCM Finalized: false")); - assertTrue(output.contains("Datanodes finalized: 1")); - assertTrue(output.contains("Total Datanodes: 3")); - assertTrue(output.contains("Should Finalize: true")); - verify(scmClient).queryUpgradeStatus(); + assertTrue(output.contains("Upgrade status")); + assertTrue(output.contains("OM Finalized? false")); + assertTrue(output.contains("SCM Finalized? false")); + assertTrue(output.contains("Datanodes finalized: 1/3")); + verify(omClient).queryUpgradeStatus(); + } + + @Test + public void testStatusCommandPropagatesException() throws Exception { + when(omClient.queryUpgradeStatus()).thenThrow(new IOException("OM unavailable")); + new CommandLine(cmd).parseArgs(); + assertThrows(IOException.class, () -> cmd.call()); + } + + @Test + public void testNonZduServerPrintsErrorAndReturnsNonZero() throws Exception { + when(omClient.getServiceInfo()).thenReturn(serviceInfoWithVersion(OzoneManagerVersion.DEFAULT_VERSION)); + + new CommandLine(cmd).parseArgs(); + assertEquals(1, cmd.call()); + + String errOutput = errContent.toString(DEFAULT_ENCODING); + assertTrue(errOutput.contains("OM does not support zero downtime upgrade")); + verify(omClient, never()).queryUpgradeStatus(); + } + + private ServiceInfoEx serviceInfoWithVersion(OzoneManagerVersion version) { + ServiceInfo serviceInfo = new ServiceInfo.Builder() + .setNodeType(HddsProtos.NodeType.OM) + .setHostname("localhost") + .setOmVersion(version) + .build(); + return new ServiceInfoEx(Collections.singletonList(serviceInfo), "", Collections.emptyList()); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 23e1d3c83822..b324fa766655 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -261,6 +261,7 @@ public static boolean isReadOnly(OMRequest omRequest) { case GetObjectTagging: case GetQuotaRepairStatus: case StartQuotaRepair: + case QueryUpgradeStatus: return true; case CreateVolume: case SetVolumeProperty: @@ -455,6 +456,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) { // Quota repair lifecycle request should be initiated by the leader case DBUpdates: // We are currently only interested on the leader DB info case UnknownCommand: + case QueryUpgradeStatus: return false; case EchoRPC: return omRequest.getEchoRPCRequest().getReadOnly(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 30b0c1404021..4830b5f6402d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -507,6 +507,14 @@ ListOpenFilesResult listOpenFiles(String path, int maxKeys, String contToken) */ void finalizeUpgrade() throws IOException; + /** + * Returns the upgrade status of the cluster. This call is received by OM which will in turn query SCM to get the + * status of it and the datanodes, and return the combined status to the caller. + * @return QueryUpgradeStatusResponse containing details of the overall cluster state + * @throws IOException If any error occurs. + */ + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse queryUpgradeStatus() throws IOException; + /** * Queries the current status of finalization. * This method when called, returns the status messages from the finalization diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index be95623fb901..27b0e4056bd5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2073,6 +2073,18 @@ public StatusAndMessages queryUpgradeFinalizationProgress( ); } + @Override + public OzoneManagerProtocolProtos.QueryUpgradeStatusResponse queryUpgradeStatus() throws IOException { + OzoneManagerProtocolProtos.QueryUpgradeStatusRequest + req = OzoneManagerProtocolProtos.QueryUpgradeStatusRequest.newBuilder().build(); + + OMRequest omRequest = createOMRequest(Type.QueryUpgradeStatus) + .setQueryUpgradeStatusRequest(req) + .build(); + + return handleError(submitRequest(omRequest)).getQueryUpgradeStatusResponse(); + } + /** * Get a valid Delegation Token. * diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 7efd46983bc9..a2d27329fc0f 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -158,6 +158,7 @@ enum Type { DeleteObjectTagging = 142; SubmitSnapshotDiff = 143; StartFinalizeUpgrade = 144; + QueryUpgradeStatus = 145; } enum SafeMode { @@ -217,7 +218,7 @@ message OMRequest { optional ServiceListRequest serviceListRequest = 51; optional DBUpdatesRequest dbUpdatesRequest = 53; optional FinalizeUpgradeRequest finalizeUpgradeRequest = 54 [deprecated = true]; - optional FinalizeUpgradeProgressRequest finalizeUpgradeProgressRequest = 55; + optional FinalizeUpgradeProgressRequest finalizeUpgradeProgressRequest = 55 [deprecated = true]; optional PrepareRequest prepareRequest = 56; optional PrepareStatusRequest prepareStatusRequest = 57; optional CancelPrepareRequest cancelPrepareRequest = 58; @@ -311,6 +312,7 @@ message OMRequest { optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144; optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 145; + optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 146; } message OMResponse { @@ -361,7 +363,7 @@ message OMResponse { optional ServiceListResponse ServiceListResponse = 51; optional DBUpdatesResponse dbUpdatesResponse = 52; optional FinalizeUpgradeResponse finalizeUpgradeResponse = 54 [deprecated = true]; - optional FinalizeUpgradeProgressResponse finalizeUpgradeProgressResponse = 55; + optional FinalizeUpgradeProgressResponse finalizeUpgradeProgressResponse = 55 [deprecated = true]; optional PrepareResponse prepareResponse = 56; optional PrepareStatusResponse prepareStatusResponse = 57; optional CancelPrepareResponse cancelPrepareResponse = 58; @@ -447,6 +449,7 @@ message OMResponse { optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143; optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 144; + optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 145; } enum Status { @@ -1640,6 +1643,15 @@ message StartFinalizeUpgradeRequest { message StartFinalizeUpgradeResponse { } +message QueryUpgradeStatusRequest { +} + +message QueryUpgradeStatusResponse { + optional bool omFinalized = 1; + optional hadoop.hdds.UpgradeStatus hddsStatus = 2; +} + +// deprecated, use QueryUpgradeStatusRequest/Response instead. Retained for wire compatibility. message FinalizeUpgradeProgressRequest { // Ignored by OM; retained for wire compatibility. required string upgradeClientId = 1; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8d99f122e49c..6010eaa47975 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -309,6 +309,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExtendedUserAccessIdInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.QueryUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantState; @@ -3648,6 +3649,16 @@ public void finalizeUpgrade() throws IOException { throw new UnsupportedOperationException(); } + @Override + public QueryUpgradeStatusResponse queryUpgradeStatus() throws IOException { + HddsProtos.UpgradeStatus scmStatus = scmClient.getContainerClient().queryUpgradeStatus(); + + return QueryUpgradeStatusResponse.newBuilder() + .setOmFinalized(!versionManager.needsFinalization()) + .setHddsStatus(scmStatus) + .build(); + } + @Override public StatusAndMessages queryUpgradeFinalizationProgress( String unusedUpgradeClientId, boolean unusedTakeover, boolean unusedReadonly) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 738eaa4cad8a..5b6f46235395 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -398,6 +398,11 @@ public OMResponse handleReadRequest(OMRequest request) { getObjectTagging(request.getGetObjectTaggingRequest()); responseBuilder.setGetObjectTaggingResponse(getObjectTaggingResponse); break; + case QueryUpgradeStatus: + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse queryUpgradeStatusResponse = + getOzoneManager().queryUpgradeStatus(); + responseBuilder.setQueryUpgradeStatusResponse(queryUpgradeStatusResponse); + break; default: responseBuilder.setSuccess(false); responseBuilder.setMessage("Unrecognized Command Type: " + cmdType); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java index 5424a970dc78..6e99f949c154 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java @@ -317,6 +317,38 @@ public void testWriteRequestExceptionAuditLog() { "Audit message should contain Command CreateVolume, got: " + formatted); } + @Test + public void testQueryUpgradeStatusDispatch() throws IOException { + OzoneManagerRequestHandler handler = getRequestHandler(10); + OzoneManager ozoneManager = handler.getOzoneManager(); + + HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setShouldFinalize(false) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .build(); + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse expected = + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() + .setOmFinalized(true) + .setHddsStatus(hddsStatus) + .build(); + Mockito.when(ozoneManager.queryUpgradeStatus()).thenReturn(expected); + + OzoneManagerProtocolProtos.OMRequest request = + OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.QueryUpgradeStatus) + .setClientId("test-client") + .build(); + + OzoneManagerProtocolProtos.OMResponse response = handler.handleReadRequest(request); + + Assertions.assertTrue(response.getSuccess()); + Assertions.assertTrue(response.hasQueryUpgradeStatusResponse()); + Assertions.assertEquals(expected, response.getQueryUpgradeStatusResponse()); + Mockito.verify(ozoneManager).queryUpgradeStatus(); + } + @Test public void testSnapshotDiffRoutingUsesCorrectServerMethodBasedOnOptionalFlags() throws IOException {