diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java index 9f0b828a24..139076a7e6 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java @@ -13,7 +13,7 @@ * * Portions Copyright 2014 The Apache Software Foundation * Copyright 2015-2016 ForgeRock AS. - * Portions Copyright 2023-2025 3A Systems, LLC + * Portions Copyright 2023-2026 3A Systems, LLC */ package org.opends.server.backends.pluggable; @@ -102,6 +102,7 @@ import org.opends.server.backends.pluggable.DN2ID.TreeVisitor; import org.opends.server.backends.pluggable.ImportLDIFReader.EntryInformation; import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool.MemoryBuffer; +import org.opends.server.backends.pluggable.spi.AccessMode; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Importer; import org.opends.server.backends.pluggable.spi.ReadOperation; @@ -118,6 +119,8 @@ import org.opends.server.types.DirectoryException; import org.opends.server.types.Entry; import org.opends.server.types.InitializationException; +import org.opends.server.types.ExistingFileBehavior; +import org.opends.server.types.LDIFExportConfig; import org.opends.server.types.LDIFImportConfig; import org.opends.server.types.LDIFImportResult; @@ -156,6 +159,8 @@ static class StrategyImpl implements ImportStrategy private static final String PHASE1_IMPORTER_THREAD_NAME = "PHASE1-IMPORTER-%d"; + private static final String PHASE1_MIGRATOR_THREAD_NAME = "PHASE1-MIGRATOR-%d"; + private static final String PHASE2_IMPORTER_THREAD_NAME = "PHASE2-IMPORTER-%d"; private static final String SORTER_THREAD_NAME = "PHASE1-SORTER-%d"; @@ -190,6 +195,23 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) { logger.info(NOTE_IMPORT_STARTING, DirectoryServer.getVersionString(), BUILD_ID, REVISION); + // A partial import (include and/or exclude branches) only replaces the + // imported branches: existing entries outside of the imported scope must + // be preserved. Snapshot them into temporary LDIF files before the + // import deletes the entry containers, so that they can be fed through + // the import pipeline together with the user-provided LDIF file. + final List migrationFiles; + try + { + migrationFiles = exportEntriesToPreserve(importConfig); + } + catch (Exception e) + { + throw new ExecutionException(e); + } + + try + { final long startTime = System.currentTimeMillis(); final int maxThreadCount = importConfig.getThreadCount() == 0 ? getDefaultNumberOfThread() @@ -212,7 +234,34 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) final AbstractTwoPhaseImportStrategy importStrategy = new ExternalSortAndImportStrategy(entryContainers, dbStorage, tempDir, bufferPool, sorter); importer = new OnDiskMergeImporter(PHASE2_IMPORTER_THREAD_NAME, importStrategy); - importer.doImport(source); + if (migrationFiles.isEmpty()) + { + importer.doImport(source); + } + else + { + final List migrationPaths = new ArrayList<>(migrationFiles.size()); + for (File migrationFile : migrationFiles) + { + migrationPaths.add(migrationFile.getAbsolutePath()); + } + final LDIFImportConfig migrationConfig = new LDIFImportConfig(migrationPaths); + try + { + // Entries come straight from this backend: no filtering and no + // schema validation must be applied to them. + migrationConfig.setValidateSchema(false); + try (final LDIFReaderSource migrationSource = + new LDIFReaderSource(rootContainer, migrationConfig, PHASE1_MIGRATOR_THREAD_NAME, threadCount)) + { + importer.doImport(new CompositeSource(Arrays.asList((Source) migrationSource, source))); + } + } + finally + { + migrationConfig.close(); + } + } } finally { @@ -246,6 +295,150 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) { throw new ExecutionException(e); } + } + finally + { + for (File migrationFile : migrationFiles) + { + migrationFile.delete(); + } + } + } + + /** + * Exports the entries which must survive a partial import to temporary + * LDIF files, mimicking the migration performed by the previous generation + * importer: when include and/or exclude branches are configured, only the + * imported branches are replaced, so existing entries outside of the + * include branches, as well as existing entries below the exclude branches + * of an imported branch, are re-imported from the snapshot files. + * + * @return the temporary LDIF files containing the entries to preserve + */ + private List exportEntriesToPreserve(LDIFImportConfig importConfig) throws Exception + { + if (importConfig.clearBackend()) + { + return Collections.emptyList(); + } + final Set includeBranches = importConfig.getIncludeBranches(); + final Set excludeBranches = importConfig.getExcludeBranches(); + if (includeBranches.isEmpty() && excludeBranches.isEmpty()) + { + return Collections.emptyList(); + } + + final List migrationFiles = new ArrayList<>(); + boolean storageOpen = false; + try + { + for (EntryContainer entryContainer : rootContainer.getEntryContainers()) + { + final DN baseDN = entryContainer.getBaseDN(); + if (excludeBranches.contains(baseDN)) + { + // This entire base DN is excluded: the import does not touch it. + continue; + } + + final List includesUnderBase = new ArrayList<>(); + for (DN includeBranch : includeBranches) + { + if (includeBranch.isSubordinateOrEqualTo(baseDN)) + { + includesUnderBase.add(includeBranch); + } + } + if (!includeBranches.isEmpty() && includesUnderBase.isEmpty()) + { + // Nothing is imported under this base DN: the import does not touch it. + continue; + } + + // Everything under this base DN which is not below an include branch + // is replaced by the import, unless the base DN itself is included. + final boolean wholeBaseReplaced = includesUnderBase.isEmpty() || includesUnderBase.contains(baseDN); + if (!wholeBaseReplaced) + { + if (!storageOpen) + { + rootContainer.getStorage().open(AccessMode.READ_ONLY); + storageOpen = true; + } + logger.info(NOTE_IMPORT_MIGRATION_START, "existing", baseDN); + migrationFiles.add( + exportBranches(Collections.singletonList(baseDN), new ArrayList<>(includesUnderBase))); + } + + // Existing entries below exclude branches located inside the + // replaced scope must be preserved as well. + final List excludedToPreserve = new ArrayList<>(); + for (DN excludeBranch : excludeBranches) + { + if (excludeBranch.isSubordinateOrEqualTo(baseDN) + && (wholeBaseReplaced || isUnderAnyOf(excludeBranch, includesUnderBase))) + { + excludedToPreserve.add(excludeBranch); + } + } + if (!excludedToPreserve.isEmpty()) + { + if (!storageOpen) + { + rootContainer.getStorage().open(AccessMode.READ_ONLY); + storageOpen = true; + } + logger.info(NOTE_IMPORT_MIGRATION_START, "excluded", baseDN); + migrationFiles.add(exportBranches(excludedToPreserve, Collections. emptyList())); + } + } + return migrationFiles; + } + catch (Exception e) + { + for (File migrationFile : migrationFiles) + { + migrationFile.delete(); + } + throw e; + } + finally + { + if (storageOpen) + { + rootContainer.getStorage().close(); + } + } + } + + private static boolean isUnderAnyOf(DN dn, List branches) + { + for (DN branch : branches) + { + if (dn.isSubordinateOrEqualTo(branch)) + { + return true; + } + } + return false; + } + + private File exportBranches(List includeBranches, List excludeBranches) throws Exception + { + final File migrationFile = File.createTempFile("import-migration-", ".ldif"); + final LDIFExportConfig exportConfig = + new LDIFExportConfig(migrationFile.getAbsolutePath(), ExistingFileBehavior.OVERWRITE); + exportConfig.setIncludeBranches(includeBranches); + exportConfig.setExcludeBranches(excludeBranches); + try + { + new ExportJob(exportConfig).exportLDIF(rootContainer); + } + finally + { + exportConfig.close(); + } + return migrationFile; } private static int getDefaultNumberOfThread() @@ -644,6 +837,45 @@ interface EntryProcessor boolean isCancelled(); } + /** Chains {@link Source}s, processing them sequentially. */ + private static final class CompositeSource implements Source + { + private final List sources; + + CompositeSource(final List sources) + { + this.sources = sources; + } + + @Override + public void processAllEntries(EntryProcessor processor) throws InterruptedException, ExecutionException + { + for (Source source : sources) + { + source.processAllEntries(processor); + } + } + + @Override + public boolean isCancelled() + { + for (Source source : sources) + { + if (source.isCancelled()) + { + return true; + } + } + return false; + } + + @Override + public void close() + { + // The chained sources are closed by their respective owners. + } + } + /** Extract LDAP {@link Entry}s from an LDIF file. */ private static final class LDIFReaderSource implements Source { diff --git a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties index 7337a24487..fde2100756 100644 --- a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties +++ b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties @@ -12,6 +12,7 @@ # # Copyright 2006-2010 Sun Microsystems, Inc. # Portions Copyright 2011-2016 ForgeRock AS. +# Portions Copyright 2026 3A Systems, LLC # @@ -1104,3 +1105,4 @@ ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_ADD_MECHANISM_611=An error occurred while a Service Discovery Mechanism '%s' : %s ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_INIT_MECHANISM_614=Service Discovery Mechanism '%s' initialization failed : %s ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_LISTENER_615=Registering Service Discovery Manager's listener failed : %s +NOTE_IMPORT_MIGRATION_START_616=Migrating %s entries for base DN %s so that they are preserved by the partial import diff --git a/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java b/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java index 9134f70460..00467aa853 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java @@ -13,6 +13,7 @@ * * Copyright 2006-2009 Sun Microsystems, Inc. * Portions Copyright 2011-2016 ForgeRock AS. + * Portions Copyright 2026 3A Systems, LLC */ package org.opends.server.tasks; @@ -347,7 +348,7 @@ public Object[][] createBadData() throws Exception * @param taskEntry The task entry. * @param expectedState The expected completion state of the task. */ - @Test(dataProvider = "importexport", groups = "slow") + @Test(dataProvider = "importexport") public void testImportExport(Entry taskEntry, TaskState expectedState) throws Exception {