From 5ef6be1b81e7472f33156e2afefc5a41ee818369 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Sat, 4 Jul 2026 13:55:33 +0300 Subject: [PATCH] Restore partial import semantics for include/exclude branches Since the import was rewritten around OnDiskMergeImporter, a partial import (import-ldif with --includeBranch and/or --excludeBranch) silently destroyed all existing entries outside of the imported scope: the importer deletes the whole entry container before rebuilding it from the filtered LDIF content, and then aborts on the orphan check because the parent of the included branch is gone ("The import has been aborted because the entry ... does not have a parent entry"). The 2.x importer used to migrate those entries into the new container (MigrateExistingTask/MigrateExcludedTask). Restore that behaviour in an architecture-compatible way: before the import starts, snapshot the entries which must survive - entries outside of the include branches, plus entries below exclude branches of an imported branch - into temporary LDIF files using the regular export machinery, then feed them through the import pipeline together with the user-provided LDIF via a composite source. Containers not touched by the import are left alone, as before. Move TestImportAndExport.testImportExport out of the "slow" group into the default build. --- .../pluggable/OnDiskMergeImporter.java | 236 +++++++++++++++++- .../org/opends/messages/backend.properties | 2 + .../server/tasks/TestImportAndExport.java | 3 +- 3 files changed, 238 insertions(+), 3 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java index 9f0b828a24..139076a7e6 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java @@ -13,7 +13,7 @@ * * Portions Copyright 2014 The Apache Software Foundation * Copyright 2015-2016 ForgeRock AS. - * Portions Copyright 2023-2025 3A Systems, LLC + * Portions Copyright 2023-2026 3A Systems, LLC */ package org.opends.server.backends.pluggable; @@ -102,6 +102,7 @@ import org.opends.server.backends.pluggable.DN2ID.TreeVisitor; import org.opends.server.backends.pluggable.ImportLDIFReader.EntryInformation; import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool.MemoryBuffer; +import org.opends.server.backends.pluggable.spi.AccessMode; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Importer; import org.opends.server.backends.pluggable.spi.ReadOperation; @@ -118,6 +119,8 @@ import org.opends.server.types.DirectoryException; import org.opends.server.types.Entry; import org.opends.server.types.InitializationException; +import org.opends.server.types.ExistingFileBehavior; +import org.opends.server.types.LDIFExportConfig; import org.opends.server.types.LDIFImportConfig; import org.opends.server.types.LDIFImportResult; @@ -156,6 +159,8 @@ static class StrategyImpl implements ImportStrategy private static final String PHASE1_IMPORTER_THREAD_NAME = "PHASE1-IMPORTER-%d"; + private static final String PHASE1_MIGRATOR_THREAD_NAME = "PHASE1-MIGRATOR-%d"; + private static final String PHASE2_IMPORTER_THREAD_NAME = "PHASE2-IMPORTER-%d"; private static final String SORTER_THREAD_NAME = "PHASE1-SORTER-%d"; @@ -190,6 +195,23 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) { logger.info(NOTE_IMPORT_STARTING, DirectoryServer.getVersionString(), BUILD_ID, REVISION); + // A partial import (include and/or exclude branches) only replaces the + // imported branches: existing entries outside of the imported scope must + // be preserved. Snapshot them into temporary LDIF files before the + // import deletes the entry containers, so that they can be fed through + // the import pipeline together with the user-provided LDIF file. + final List migrationFiles; + try + { + migrationFiles = exportEntriesToPreserve(importConfig); + } + catch (Exception e) + { + throw new ExecutionException(e); + } + + try + { final long startTime = System.currentTimeMillis(); final int maxThreadCount = importConfig.getThreadCount() == 0 ? getDefaultNumberOfThread() @@ -212,7 +234,34 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) final AbstractTwoPhaseImportStrategy importStrategy = new ExternalSortAndImportStrategy(entryContainers, dbStorage, tempDir, bufferPool, sorter); importer = new OnDiskMergeImporter(PHASE2_IMPORTER_THREAD_NAME, importStrategy); - importer.doImport(source); + if (migrationFiles.isEmpty()) + { + importer.doImport(source); + } + else + { + final List migrationPaths = new ArrayList<>(migrationFiles.size()); + for (File migrationFile : migrationFiles) + { + migrationPaths.add(migrationFile.getAbsolutePath()); + } + final LDIFImportConfig migrationConfig = new LDIFImportConfig(migrationPaths); + try + { + // Entries come straight from this backend: no filtering and no + // schema validation must be applied to them. + migrationConfig.setValidateSchema(false); + try (final LDIFReaderSource migrationSource = + new LDIFReaderSource(rootContainer, migrationConfig, PHASE1_MIGRATOR_THREAD_NAME, threadCount)) + { + importer.doImport(new CompositeSource(Arrays.asList((Source) migrationSource, source))); + } + } + finally + { + migrationConfig.close(); + } + } } finally { @@ -246,6 +295,150 @@ public LDIFImportResult importLDIF(LDIFImportConfig importConfig) { throw new ExecutionException(e); } + } + finally + { + for (File migrationFile : migrationFiles) + { + migrationFile.delete(); + } + } + } + + /** + * Exports the entries which must survive a partial import to temporary + * LDIF files, mimicking the migration performed by the previous generation + * importer: when include and/or exclude branches are configured, only the + * imported branches are replaced, so existing entries outside of the + * include branches, as well as existing entries below the exclude branches + * of an imported branch, are re-imported from the snapshot files. + * + * @return the temporary LDIF files containing the entries to preserve + */ + private List exportEntriesToPreserve(LDIFImportConfig importConfig) throws Exception + { + if (importConfig.clearBackend()) + { + return Collections.emptyList(); + } + final Set includeBranches = importConfig.getIncludeBranches(); + final Set excludeBranches = importConfig.getExcludeBranches(); + if (includeBranches.isEmpty() && excludeBranches.isEmpty()) + { + return Collections.emptyList(); + } + + final List migrationFiles = new ArrayList<>(); + boolean storageOpen = false; + try + { + for (EntryContainer entryContainer : rootContainer.getEntryContainers()) + { + final DN baseDN = entryContainer.getBaseDN(); + if (excludeBranches.contains(baseDN)) + { + // This entire base DN is excluded: the import does not touch it. + continue; + } + + final List includesUnderBase = new ArrayList<>(); + for (DN includeBranch : includeBranches) + { + if (includeBranch.isSubordinateOrEqualTo(baseDN)) + { + includesUnderBase.add(includeBranch); + } + } + if (!includeBranches.isEmpty() && includesUnderBase.isEmpty()) + { + // Nothing is imported under this base DN: the import does not touch it. + continue; + } + + // Everything under this base DN which is not below an include branch + // is replaced by the import, unless the base DN itself is included. + final boolean wholeBaseReplaced = includesUnderBase.isEmpty() || includesUnderBase.contains(baseDN); + if (!wholeBaseReplaced) + { + if (!storageOpen) + { + rootContainer.getStorage().open(AccessMode.READ_ONLY); + storageOpen = true; + } + logger.info(NOTE_IMPORT_MIGRATION_START, "existing", baseDN); + migrationFiles.add( + exportBranches(Collections.singletonList(baseDN), new ArrayList<>(includesUnderBase))); + } + + // Existing entries below exclude branches located inside the + // replaced scope must be preserved as well. + final List excludedToPreserve = new ArrayList<>(); + for (DN excludeBranch : excludeBranches) + { + if (excludeBranch.isSubordinateOrEqualTo(baseDN) + && (wholeBaseReplaced || isUnderAnyOf(excludeBranch, includesUnderBase))) + { + excludedToPreserve.add(excludeBranch); + } + } + if (!excludedToPreserve.isEmpty()) + { + if (!storageOpen) + { + rootContainer.getStorage().open(AccessMode.READ_ONLY); + storageOpen = true; + } + logger.info(NOTE_IMPORT_MIGRATION_START, "excluded", baseDN); + migrationFiles.add(exportBranches(excludedToPreserve, Collections. emptyList())); + } + } + return migrationFiles; + } + catch (Exception e) + { + for (File migrationFile : migrationFiles) + { + migrationFile.delete(); + } + throw e; + } + finally + { + if (storageOpen) + { + rootContainer.getStorage().close(); + } + } + } + + private static boolean isUnderAnyOf(DN dn, List branches) + { + for (DN branch : branches) + { + if (dn.isSubordinateOrEqualTo(branch)) + { + return true; + } + } + return false; + } + + private File exportBranches(List includeBranches, List excludeBranches) throws Exception + { + final File migrationFile = File.createTempFile("import-migration-", ".ldif"); + final LDIFExportConfig exportConfig = + new LDIFExportConfig(migrationFile.getAbsolutePath(), ExistingFileBehavior.OVERWRITE); + exportConfig.setIncludeBranches(includeBranches); + exportConfig.setExcludeBranches(excludeBranches); + try + { + new ExportJob(exportConfig).exportLDIF(rootContainer); + } + finally + { + exportConfig.close(); + } + return migrationFile; } private static int getDefaultNumberOfThread() @@ -644,6 +837,45 @@ interface EntryProcessor boolean isCancelled(); } + /** Chains {@link Source}s, processing them sequentially. */ + private static final class CompositeSource implements Source + { + private final List sources; + + CompositeSource(final List sources) + { + this.sources = sources; + } + + @Override + public void processAllEntries(EntryProcessor processor) throws InterruptedException, ExecutionException + { + for (Source source : sources) + { + source.processAllEntries(processor); + } + } + + @Override + public boolean isCancelled() + { + for (Source source : sources) + { + if (source.isCancelled()) + { + return true; + } + } + return false; + } + + @Override + public void close() + { + // The chained sources are closed by their respective owners. + } + } + /** Extract LDAP {@link Entry}s from an LDIF file. */ private static final class LDIFReaderSource implements Source { diff --git a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties index 7337a24487..fde2100756 100644 --- a/opendj-server-legacy/src/messages/org/opends/messages/backend.properties +++ b/opendj-server-legacy/src/messages/org/opends/messages/backend.properties @@ -12,6 +12,7 @@ # # Copyright 2006-2010 Sun Microsystems, Inc. # Portions Copyright 2011-2016 ForgeRock AS. +# Portions Copyright 2026 3A Systems, LLC # @@ -1104,3 +1105,4 @@ ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_ADD_MECHANISM_611=An error occurred while a Service Discovery Mechanism '%s' : %s ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_INIT_MECHANISM_614=Service Discovery Mechanism '%s' initialization failed : %s ERR_SERVICE_DISCOVERY_CONFIG_MANAGER_LISTENER_615=Registering Service Discovery Manager's listener failed : %s +NOTE_IMPORT_MIGRATION_START_616=Migrating %s entries for base DN %s so that they are preserved by the partial import diff --git a/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java b/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java index 9134f70460..00467aa853 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/tasks/TestImportAndExport.java @@ -13,6 +13,7 @@ * * Copyright 2006-2009 Sun Microsystems, Inc. * Portions Copyright 2011-2016 ForgeRock AS. + * Portions Copyright 2026 3A Systems, LLC */ package org.opends.server.tasks; @@ -347,7 +348,7 @@ public Object[][] createBadData() throws Exception * @param taskEntry The task entry. * @param expectedState The expected completion state of the task. */ - @Test(dataProvider = "importexport", groups = "slow") + @Test(dataProvider = "importexport") public void testImportExport(Entry taskEntry, TaskState expectedState) throws Exception {