Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,20 @@

package org.apache.hadoop.ozone.debug.datanode.container.analyze;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.stream.Stream;
import org.apache.hadoop.hdds.cli.AbstractSubcommand;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.shell.ListLimitOptions;
import picocli.CommandLine;
import picocli.CommandLine.Command;

Expand All @@ -34,65 +42,134 @@
*/
@Command(
name = "analyze",
description = "Analyze container consistency between on-disk container " +
"directories on this DataNode and SCM metadata. Must be run locally on a DataNode.")
description = {
"Analyze container consistency between on-disk container directories on this DataNode and SCM metadata.",
"Must be run locally on a DataNode.",
"",
"Reports:",
" Duplicate container directories: same containerID found on more than one volume.",
" Orphan containers (requires --scm-db): present on disk but not present in SCM metadata.",
" Containers marked DELETED in SCM but present on disk (requires --scm-db).",
"",
"Each reported occurrence includes container directory path(s), size and an on-disk metadata status:",
" MISSING_METADATA: metadata/{containerId}.container does not exist.",
" INVALID_METADATA: metadata file exists but cannot be parsed, or the containerID in the",
" file does not match the directory name.",
" VALID: metadata file is present, parses correctly, and its containerID matches the directory name."
})
public class AnalyzeSubcommand extends AbstractSubcommand implements Callable<Void> {
@CommandLine.Option(names = {"--count"},
defaultValue = "20",
description = "Number of containers to display")
private int count;
@CommandLine.Mixin
private ListLimitOptions listOptions;

@CommandLine.Option(names = {"--scm-db"},
description = "Path to an offline scm.db directory, or its parent metadata directory.")
private File scmDb;

@Override
public Void call() throws Exception {
if (count < 1) {
throw new IOException("Count must be an integer greater than 0.");
}
validateOptions();
OzoneConfiguration conf = getOzoneConf();
ContainerScanResult scanResult = ContainerDirectoryScanner.scan(conf);
Map<Long, List<ContainerDiskOccurrence>> enrichedDuplicates =
ContainerDirectoryScanner.enrichDuplicates(scanResult.getDuplicates());

// TODO: SCM metadata lookup from --scm-db when provided.
// TODO: For each id in scanResult.getSingles().keySet() classified NOT_IN_SCM or DELETED:
// enrichOccurrence(id, scanResult.getSingles().get(id)) and report.
// TODO: For each id in enrichedDuplicates.keySet() classified NOT_IN_SCM or DELETED:
// enrichedDuplicates.get(id) is already enriched — just report.
if (scmDb != null) {
findOrphanAndDeletedButPresentContainers(conf, scanResult, enrichedDuplicates);
} else {
out().println("To identify orphan containers (wrt SCM) and containers that are marked as DELETED in SCM but"
+ " exist in the datanode's current directory, provide the SCM database path using the --scm-db option."
);
}

printDuplicates(enrichedDuplicates);
printVolumeScanErrors(scanResult.getVolumeScanErrors());
return null;
}

private void printDuplicates(Map<Long, List<ContainerDiskOccurrence>> duplicates) {
long totalDuplicateIds = duplicates.size();
out().printf("Number of containers with duplicate container directories on this DataNode: %d%n", totalDuplicateIds);
/**
* Validate CLI options before starting the on-disk DN scan.
* {@link ListLimitOptions#getLimit()} is also called from
* {@link #printContainerOccurrenceReport(String, Map)}, but validating here fails fast
* before the DN volume scan and SCM DB lookup.
*/
private void validateOptions() {
listOptions.getLimit();
}

private void findOrphanAndDeletedButPresentContainers(OzoneConfiguration conf, ContainerScanResult scanResult,
Map<Long, List<ContainerDiskOccurrence>> enrichedDuplicates) throws IOException {
Map<Long, List<ContainerDiskOccurrence>> enrichedOrphanContainers = new HashMap<>();
Map<Long, List<ContainerDiskOccurrence>> enrichedDeletedButPresent = new HashMap<>();

try (ScmContainerMetadataReader reader = new ScmContainerMetadataReader(conf, scmDb)) {
Set<Long> containerIds = new HashSet<>(scanResult.getSingles().keySet());
containerIds.addAll(enrichedDuplicates.keySet());

for (long containerId : containerIds) {
Optional<ScmContainerMetadataReader.ScmContainerClassification> classification = reader.classify(containerId);
if (!classification.isPresent()) {
continue;
}
List<ContainerDiskOccurrence> occurrences = enrichedDuplicates.get(containerId);
if (occurrences == null) {
String path = scanResult.getSingles().get(containerId);
occurrences = Collections.singletonList(ContainerDirectoryScanner.enrichOccurrence(containerId, path));
}
if (classification.get() == ScmContainerMetadataReader.ScmContainerClassification.NOT_IN_SCM) {
enrichedOrphanContainers.put(containerId, occurrences);
} else {
enrichedDeletedButPresent.put(containerId, occurrences);
}
}
}

printContainerOccurrenceReport("Number of orphan containers(wrt SCM) on this DataNode: %d%n",
enrichedOrphanContainers);
printContainerOccurrenceReport(
"Number of containers marked DELETED in SCM but present on disk on this DataNode: %d%n",
enrichedDeletedButPresent);
}

if (totalDuplicateIds == 0) {
private void printContainerOccurrenceReport(String countFormat,
Map<Long, List<ContainerDiskOccurrence>> containersById) {
long total = containersById.size();
out().printf(countFormat, total);
if (total == 0) {
return;
}

if (totalDuplicateIds > count) {
out().printf("Showing first %d:%n", count);
Stream<Map.Entry<Long, List<ContainerDiskOccurrence>>> stream =
containersById.entrySet().stream().sorted(Map.Entry.comparingByKey());
if (!listOptions.isAll()) {
int limit = listOptions.getLimit();
if (total > limit) {
out().printf("Showing first %d:%n", limit);
}
stream = stream.limit(limit);
}
stream.forEach(entry -> printContainerEntry(entry.getKey(), entry.getValue()));
}

private void printContainerEntry(long containerId, List<ContainerDiskOccurrence> occurrences) {
out().printf("Container %d (%d occurrence%s):%n",
containerId,
occurrences.size(),
occurrences.size() == 1 ? "" : "s");
for (ContainerDiskOccurrence occurrence : occurrences) {
out().printf(" path=%s%n", occurrence.getContainerPath());
if (occurrence.isSizeKnown()) {
out().printf(" status=%s size=%d bytes%n", occurrence.getStatus(), occurrence.getSizeBytes());
} else {
out().printf(" status=%s size=unavailable (failed to compute directory size)%n", occurrence.getStatus());
}
out().println();
}
}

duplicates.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.limit(count)
.forEach(entry -> {
long containerId = entry.getKey();
List<ContainerDiskOccurrence> occurrences = entry.getValue();
out().printf("Container %d (%d occurrences):%n", containerId, occurrences.size());
for (ContainerDiskOccurrence o : occurrences) {
out().printf(" path=%s%n", o.getContainerPath());
if (o.isSizeKnown()) {
out().printf(" status=%s size=%d bytes%n", o.getStatus(), o.getSizeBytes());
} else {
out().printf(" status=%s size=unavailable (failed to compute directory size)%n",
o.getStatus());
}
out().println();
}
});
private void printDuplicates(Map<Long, List<ContainerDiskOccurrence>> duplicates) {
printContainerOccurrenceReport(
"Number of containers with duplicate container directories on this DataNode: %d%n",
duplicates);
}

private void printVolumeScanErrors(List<String> volumeScanErrors) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.ozone.debug.datanode.container.analyze;

import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS;

import java.io.File;
import java.io.IOException;
import java.util.Objects;
import java.util.Optional;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.db.CodecException;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType;
import org.apache.hadoop.ozone.OzoneConsts;

/**
* Read-only lookup of container metadata from {@code scm.db}.
*/
public final class ScmContainerMetadataReader implements AutoCloseable {

private final DBStore dbStore;
private final Table<ContainerID, ContainerInfo> containerTable;

public ScmContainerMetadataReader(ConfigurationSource conf, File scmDbPath)
throws IOException {
File scmDbDir = resolveScmDbDirectory(scmDbPath);
File parentDir = scmDbDir.getParentFile();
if (parentDir == null) {
throw new IOException("SCM database directory has no parent path: " + scmDbDir);
}
try {
this.dbStore = DBStoreBuilder.newBuilder(conf, SCMDBDefinition.get(), scmDbDir.getName(),
parentDir.toPath())
.setOpenReadOnly(true)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if someone points to online live DB, it will fail, if another process holds the write lock.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. it is better if --scm-db point at an offline copy of scm.db, opening read-only does not avoid RocksDB lock contention if SCM is still running. Same I think can also be seen in ozone debug ldb scan right.

.build();
} catch (RocksDatabaseException e) {
throw new IOException("Failed to open SCM database at " + scmDbDir, e);
}
try {
this.containerTable = CONTAINERS.getTable(dbStore, CacheType.NO_CACHE);
} catch (RocksDatabaseException | CodecException e) {
dbStore.close();
throw new IOException("Failed to open scm.db containers column family at " + scmDbDir, e);
}
}

/**
* Classify a container ID against scm.db {@code containers}.
*
* @return {@link Optional#empty()} when the container is present in SCM with a
* non-DELETED lifecycle state
*/
public Optional<ScmContainerClassification> classify(long containerId) throws IOException {
try {
ContainerInfo info = containerTable.get(ContainerID.valueOf(containerId));
if (info == null) {
return Optional.of(ScmContainerClassification.NOT_IN_SCM);
}
if (info.isDeleted()) {
return Optional.of(ScmContainerClassification.DELETED);
}
return Optional.empty();
} catch (RocksDatabaseException | CodecException e) {
throw new IOException("Failed to read container " + containerId + " from scm.db", e);
}
}

static File resolveScmDbDirectory(File path) throws IOException {
Objects.requireNonNull(path, "scmDbPath");
File absolutePath = path.getAbsoluteFile();
File scmDbDir = absolutePath;
if (!OzoneConsts.SCM_DB_NAME.equals(absolutePath.getName())) {
File child = new File(absolutePath, OzoneConsts.SCM_DB_NAME);
if (child.isDirectory()) {
scmDbDir = child;
}
}
if (!scmDbDir.isDirectory()) {
throw new IOException("SCM database directory not found: " + path);
}
return scmDbDir;
}

@Override
public void close() {
if (dbStore != null) {
dbStore.close();
}
}

/**
* SCM-side classification for an on-disk container directory.
*/
enum ScmContainerClassification {
/** No record for this container ID in scm.db {@code containers}. */
NOT_IN_SCM,
/** Record exists and {@link ContainerInfo} state is DELETED. */
DELETED
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,19 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import java.util.UUID;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
Expand Down Expand Up @@ -110,4 +120,31 @@ void corruptVersionFile(File volumeRoot) throws IOException {
File versionFile = StorageVolumeUtil.getVersionFile(hddsRoot);
Files.write(versionFile.toPath(), new byte[0]);
}

/**
* Creates an offline {@code scm.db} with the given container states.
*
* @return path to the {@code scm.db} directory
*/
File createScmDb(Map<Long, HddsProtos.LifeCycleState> containerStates) throws IOException {
Path scmRoot = tempDir.resolve("scm-metadata");
Files.createDirectories(scmRoot);
DBStore dbStore = DBStoreBuilder.newBuilder(conf, SCMDBDefinition.get(), OzoneConsts.SCM_DB_NAME, scmRoot).build();
try {
Table<ContainerID, ContainerInfo> containerTable = SCMDBDefinition.CONTAINERS.getTable(dbStore);
for (Map.Entry<Long, HddsProtos.LifeCycleState> entry : containerStates.entrySet()) {
long containerId = entry.getKey();
ContainerInfo containerInfo = new ContainerInfo.Builder()
.setContainerID(containerId)
.setState(entry.getValue())
.setOwner("test")
.setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE))
.build();
containerTable.put(ContainerID.valueOf(containerId), containerInfo);
}
} finally {
dbStore.close();
}
return scmRoot.resolve(OzoneConsts.SCM_DB_NAME).toFile();
}
}
Loading